450 files changed, 34202 insertions, 10551 deletions

diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index b4e4fc0d6650..f86427591167 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -176,6 +176,8 @@ source "drivers/gpu/drm/mgag200/Kconfig"
 
 source "drivers/gpu/drm/cirrus/Kconfig"
 
+source "drivers/gpu/drm/armada/Kconfig"
+
 source "drivers/gpu/drm/rcar-du/Kconfig"
 
 source "drivers/gpu/drm/shmobile/Kconfig"
@@ -187,3 +189,5 @@ source "drivers/gpu/drm/tilcdc/Kconfig"
 source "drivers/gpu/drm/qxl/Kconfig"
 
 source "drivers/gpu/drm/msm/Kconfig"
+
+source "drivers/gpu/drm/tegra/Kconfig"
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 5af240bfd29f..cc08b845f965 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -50,10 +50,12 @@ obj-$(CONFIG_DRM_EXYNOS) +=exynos/
 obj-$(CONFIG_DRM_GMA500) += gma500/
 obj-$(CONFIG_DRM_UDL) += udl/
 obj-$(CONFIG_DRM_AST) += ast/
+obj-$(CONFIG_DRM_ARMADA) += armada/
 obj-$(CONFIG_DRM_RCAR_DU) += rcar-du/
 obj-$(CONFIG_DRM_SHMOBILE) +=shmobile/
 obj-$(CONFIG_DRM_OMAP)  += omapdrm/
 obj-$(CONFIG_DRM_TILCDC)        += tilcdc/
 obj-$(CONFIG_DRM_QXL) += qxl/
 obj-$(CONFIG_DRM_MSM) += msm/
+obj-$(CONFIG_DRM_TEGRA) += tegra/
 obj-y                   += i2c/
diff --git a/drivers/gpu/drm/armada/Kconfig b/drivers/gpu/drm/armada/Kconfig
new file mode 100644
index 000000000000..40d371521fe1
--- /dev/null
+++ b/drivers/gpu/drm/armada/Kconfig
@@ -0,0 +1,24 @@
+config DRM_ARMADA
+        tristate "DRM support for Marvell Armada SoCs"
+        depends on DRM && HAVE_CLK && ARM
+        select FB_CFB_FILLRECT
+        select FB_CFB_COPYAREA
+        select FB_CFB_IMAGEBLIT
+        select DRM_KMS_HELPER
+        help
+          Support the "LCD" controllers found on the Marvell Armada 510
+          devices.  There are two controllers on the device, each controller
+          supports graphics and video overlays.
+
+          This driver provides no built-in acceleration; acceleration is
+          performed by other IP found on the SoC.  This driver provides
+          kernel mode setting and buffer management to userspace.
+
+config DRM_ARMADA_TDA1998X
+        bool "Support TDA1998X HDMI output"
+        depends on DRM_ARMADA != n
+        depends on I2C && DRM_I2C_NXP_TDA998X = y
+        default y
+        help
+          Support the TDA1998x HDMI output device found on the Solid-Run
+          CuBox.
diff --git a/drivers/gpu/drm/armada/Makefile b/drivers/gpu/drm/armada/Makefile
new file mode 100644
index 000000000000..d6f43e06150a
--- /dev/null
+++ b/drivers/gpu/drm/armada/Makefile
@@ -0,0 +1,7 @@
+armada-y        := armada_crtc.o armada_drv.o armada_fb.o armada_fbdev.o \
+                   armada_gem.o armada_output.o armada_overlay.o \
+                   armada_slave.o
+armada-y        += armada_510.o
+armada-$(CONFIG_DEBUG_FS) += armada_debugfs.o
+
+obj-$(CONFIG_DRM_ARMADA) := armada.o
diff --git a/drivers/gpu/drm/armada/armada_510.c b/drivers/gpu/drm/armada/armada_510.c
new file mode 100644
index 000000000000..59948eff6095
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_510.c
@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * Armada 510 (aka Dove) variant support
+ */
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include "armada_crtc.h"
+#include "armada_drm.h"
+#include "armada_hw.h"
+
+static int armada510_init(struct armada_private *priv, struct device *dev)
+{
+        priv->extclk[0] = devm_clk_get(dev, "ext_ref_clk_1");
+
+        if (IS_ERR(priv->extclk[0]) && PTR_ERR(priv->extclk[0]) == -ENOENT)
+                priv->extclk[0] = ERR_PTR(-EPROBE_DEFER);
+
+        return PTR_RET(priv->extclk[0]);
+}
+
+static int armada510_crtc_init(struct armada_crtc *dcrtc)
+{
+        /* Lower the watermark so to eliminate jitter at higher bandwidths */
+        armada_updatel(0x20, (1 << 11) | 0xff, dcrtc->base + LCD_CFG_RDREG4F);
+        return 0;
+}
+
+/*
+ * Armada510 specific SCLK register selection.
+ * This gets called with sclk = NULL to test whether the mode is
+ * supportable, and again with sclk != NULL to set the clocks up for
+ * that.  The former can return an error, but the latter is expected
+ * not to.
+ *
+ * We currently are pretty rudimentary here, always selecting
+ * EXT_REF_CLK_1 for LCD0 and erroring LCD1.  This needs improvement!
+ */
+static int armada510_crtc_compute_clock(struct armada_crtc *dcrtc,
+        const struct drm_display_mode *mode, uint32_t *sclk)
+{
+        struct armada_private *priv = dcrtc->crtc.dev->dev_private;
+        struct clk *clk = priv->extclk[0];
+        int ret;
+
+        if (dcrtc->num == 1)
+                return -EINVAL;
+
+        if (IS_ERR(clk))
+                return PTR_ERR(clk);
+
+        if (dcrtc->clk != clk) {
+                ret = clk_prepare_enable(clk);
+                if (ret)
+                        return ret;
+                dcrtc->clk = clk;
+        }
+
+        if (sclk) {
+                uint32_t rate, ref, div;
+
+                rate = mode->clock * 1000;
+                ref = clk_round_rate(clk, rate);
+                div = DIV_ROUND_UP(ref, rate);
+                if (div < 1)
+                        div = 1;
+
+                clk_set_rate(clk, ref);
+                *sclk = div | SCLK_510_EXTCLK1;
+        }
+
+        return 0;
+}
+
+const struct armada_variant armada510_ops = {
+        .has_spu_adv_reg = true,
+        .spu_adv_reg = ADV_HWC32ENABLE | ADV_HWC32ARGB | ADV_HWC32BLEND,
+        .init = armada510_init,
+        .crtc_init = armada510_crtc_init,
+        .crtc_compute_clock = armada510_crtc_compute_clock,
+};
diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c
new file mode 100644
index 000000000000..d8e398275ca8
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_crtc.c
@@ -0,0 +1,1098 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *  Rewritten from the dovefb driver, and Armada510 manuals.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/clk.h>
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include "armada_crtc.h"
+#include "armada_drm.h"
+#include "armada_fb.h"
+#include "armada_gem.h"
+#include "armada_hw.h"
+
+struct armada_frame_work {
+        struct drm_pending_vblank_event *event;
+        struct armada_regs regs[4];
+        struct drm_framebuffer *old_fb;
+};
+
+enum csc_mode {
+        CSC_AUTO = 0,
+        CSC_YUV_CCIR601 = 1,
+        CSC_YUV_CCIR709 = 2,
+        CSC_RGB_COMPUTER = 1,
+        CSC_RGB_STUDIO = 2,
+};
+
+/*
+ * A note about interlacing.  Let's consider HDMI 1920x1080i.
+ * The timing parameters we have from X are:
+ *  Hact HsyA HsyI Htot  Vact VsyA VsyI Vtot
+ *  1920 2448 2492 2640  1080 1084 1094 1125
+ * Which get translated to:
+ *  Hact HsyA HsyI Htot  Vact VsyA VsyI Vtot
+ *  1920 2448 2492 2640   540  542  547  562
+ *
+ * This is how it is defined by CEA-861-D - line and pixel numbers are
+ * referenced to the rising edge of VSYNC and HSYNC.  Total clocks per
+ * line: 2640.  The odd frame, the first active line is at line 21, and
+ * the even frame, the first active line is 584.
+ *
+ * LN:    560     561     562     563             567     568    569
+ * DE:    ~~~|____________________________//__________________________
+ * HSYNC: ____|~|_____|~|_____|~|_____|~|_//__|~|_____|~|_____|~|_____
+ * VSYNC: _________________________|~~~~~~//~~~~~~~~~~~~~~~|__________
+ *  22 blanking lines.  VSYNC at 1320 (referenced to the HSYNC rising edge).
+ *
+ * LN:    1123   1124    1125      1               5       6      7
+ * DE:    ~~~|____________________________//__________________________
+ * HSYNC: ____|~|_____|~|_____|~|_____|~|_//__|~|_____|~|_____|~|_____
+ * VSYNC: ____________________|~~~~~~~~~~~//~~~~~~~~~~|_______________
+ *  23 blanking lines
+ *
+ * The Armada LCD Controller line and pixel numbers are, like X timings,
+ * referenced to the top left of the active frame.
+ *
+ * So, translating these to our LCD controller:
+ *  Odd frame, 563 total lines, VSYNC at line 543-548, pixel 1128.
+ *  Even frame, 562 total lines, VSYNC at line 542-547, pixel 2448.
+ * Note: Vsync front porch remains constant!
+ *
+ * if (odd_frame) {
+ *   vtotal = mode->crtc_vtotal + 1;
+ *   vbackporch = mode->crtc_vsync_start - mode->crtc_vdisplay + 1;
+ *   vhorizpos = mode->crtc_hsync_start - mode->crtc_htotal / 2
+ * } else {
+ *   vtotal = mode->crtc_vtotal;
+ *   vbackporch = mode->crtc_vsync_start - mode->crtc_vdisplay;
+ *   vhorizpos = mode->crtc_hsync_start;
+ * }
+ * vfrontporch = mode->crtc_vtotal - mode->crtc_vsync_end;
+ *
+ * So, we need to reprogram these registers on each vsync event:
+ *  LCD_SPU_V_PORCH, LCD_SPU_ADV_REG, LCD_SPUT_V_H_TOTAL
+ *
+ * Note: we do not use the frame done interrupts because these appear
+ * to happen too early, and lead to jitter on the display (presumably
+ * they occur at the end of the last active line, before the vsync back
+ * porch, which we're reprogramming.)
+ */
+
+void
+armada_drm_crtc_update_regs(struct armada_crtc *dcrtc, struct armada_regs *regs)
+{
+        while (regs->offset != ~0) {
+                void __iomem *reg = dcrtc->base + regs->offset;
+                uint32_t val;
+
+                val = regs->mask;
+                if (val != 0)
+                        val &= readl_relaxed(reg);
+                writel_relaxed(val | regs->val, reg);
+                ++regs;
+        }
+}
+
+#define dpms_blanked(dpms)      ((dpms) != DRM_MODE_DPMS_ON)
+
+static void armada_drm_crtc_update(struct armada_crtc *dcrtc)
+{
+        uint32_t dumb_ctrl;
+
+        dumb_ctrl = dcrtc->cfg_dumb_ctrl;
+
+        if (!dpms_blanked(dcrtc->dpms))
+                dumb_ctrl |= CFG_DUMB_ENA;
+
+        /*
+         * When the dumb interface isn't in DUMB24_RGB888_0 mode, it might
+         * be using SPI or GPIO.  If we set this to DUMB_BLANK, we will
+         * force LCD_D[23:0] to output blank color, overriding the GPIO or
+         * SPI usage.  So leave it as-is unless in DUMB24_RGB888_0 mode.
+         */
+        if (dpms_blanked(dcrtc->dpms) &&
+            (dumb_ctrl & DUMB_MASK) == DUMB24_RGB888_0) {
+                dumb_ctrl &= ~DUMB_MASK;
+                dumb_ctrl |= DUMB_BLANK;
+        }
+
+        /*
+         * The documentation doesn't indicate what the normal state of
+         * the sync signals are.  Sebastian Hesselbart kindly probed
+         * these signals on his board to determine their state.
+         *
+         * The non-inverted state of the sync signals is active high.
+         * Setting these bits makes the appropriate signal active low.
+         */
+        if (dcrtc->crtc.mode.flags & DRM_MODE_FLAG_NCSYNC)
+                dumb_ctrl |= CFG_INV_CSYNC;
+        if (dcrtc->crtc.mode.flags & DRM_MODE_FLAG_NHSYNC)
+                dumb_ctrl |= CFG_INV_HSYNC;
+        if (dcrtc->crtc.mode.flags & DRM_MODE_FLAG_NVSYNC)
+                dumb_ctrl |= CFG_INV_VSYNC;
+
+        if (dcrtc->dumb_ctrl != dumb_ctrl) {
+                dcrtc->dumb_ctrl = dumb_ctrl;
+                writel_relaxed(dumb_ctrl, dcrtc->base + LCD_SPU_DUMB_CTRL);
+        }
+}
+
+static unsigned armada_drm_crtc_calc_fb(struct drm_framebuffer *fb,
+        int x, int y, struct armada_regs *regs, bool interlaced)
+{
+        struct armada_gem_object *obj = drm_fb_obj(fb);
+        unsigned pitch = fb->pitches[0];
+        unsigned offset = y * pitch + x * fb->bits_per_pixel / 8;
+        uint32_t addr_odd, addr_even;
+        unsigned i = 0;
+
+        DRM_DEBUG_DRIVER("pitch %u x %d y %d bpp %d\n",
+                pitch, x, y, fb->bits_per_pixel);
+
+        addr_odd = addr_even = obj->dev_addr + offset;
+
+        if (interlaced) {
+                addr_even += pitch;
+                pitch *= 2;
+        }
+
+        /* write offset, base, and pitch */
+        armada_reg_queue_set(regs, i, addr_odd, LCD_CFG_GRA_START_ADDR0);
+        armada_reg_queue_set(regs, i, addr_even, LCD_CFG_GRA_START_ADDR1);
+        armada_reg_queue_mod(regs, i, pitch, 0xffff, LCD_CFG_GRA_PITCH);
+
+        return i;
+}
+
+static int armada_drm_crtc_queue_frame_work(struct armada_crtc *dcrtc,
+        struct armada_frame_work *work)
+{
+        struct drm_device *dev = dcrtc->crtc.dev;
+        unsigned long flags;
+        int ret;
+
+        ret = drm_vblank_get(dev, dcrtc->num);
+        if (ret) {
+                DRM_ERROR("failed to acquire vblank counter\n");
+                return ret;
+        }
+
+        spin_lock_irqsave(&dev->event_lock, flags);
+        if (!dcrtc->frame_work)
+                dcrtc->frame_work = work;
+        else
+                ret = -EBUSY;
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+
+        if (ret)
+                drm_vblank_put(dev, dcrtc->num);
+
+        return ret;
+}
+
+static void armada_drm_crtc_complete_frame_work(struct armada_crtc *dcrtc)
+{
+        struct drm_device *dev = dcrtc->crtc.dev;
+        struct armada_frame_work *work = dcrtc->frame_work;
+
+        dcrtc->frame_work = NULL;
+
+        armada_drm_crtc_update_regs(dcrtc, work->regs);
+
+        if (work->event)
+                drm_send_vblank_event(dev, dcrtc->num, work->event);
+
+        drm_vblank_put(dev, dcrtc->num);
+
+        /* Finally, queue the process-half of the cleanup. */
+        __armada_drm_queue_unref_work(dcrtc->crtc.dev, work->old_fb);
+        kfree(work);
+}
+
+static void armada_drm_crtc_finish_fb(struct armada_crtc *dcrtc,
+        struct drm_framebuffer *fb, bool force)
+{
+        struct armada_frame_work *work;
+
+        if (!fb)
+                return;
+
+        if (force) {
+                /* Display is disabled, so just drop the old fb */
+                drm_framebuffer_unreference(fb);
+                return;
+        }
+
+        work = kmalloc(sizeof(*work), GFP_KERNEL);
+        if (work) {
+                int i = 0;
+                work->event = NULL;
+                work->old_fb = fb;
+                armada_reg_queue_end(work->regs, i);
+
+                if (armada_drm_crtc_queue_frame_work(dcrtc, work) == 0)
+                        return;
+
+                kfree(work);
+        }
+
+        /*
+         * Oops - just drop the reference immediately and hope for
+         * the best.  The worst that will happen is the buffer gets
+         * reused before it has finished being displayed.
+         */
+        drm_framebuffer_unreference(fb);
+}
+
+static void armada_drm_vblank_off(struct armada_crtc *dcrtc)
+{
+        struct drm_device *dev = dcrtc->crtc.dev;
+
+        /*
+         * Tell the DRM core that vblank IRQs aren't going to happen for
+         * a while.  This cleans up any pending vblank events for us.
+         */
+        drm_vblank_off(dev, dcrtc->num);
+
+        /* Handle any pending flip event. */
+        spin_lock_irq(&dev->event_lock);
+        if (dcrtc->frame_work)
+                armada_drm_crtc_complete_frame_work(dcrtc);
+        spin_unlock_irq(&dev->event_lock);
+}
+
+void armada_drm_crtc_gamma_set(struct drm_crtc *crtc, u16 r, u16 g, u16 b,
+        int idx)
+{
+}
+
+void armada_drm_crtc_gamma_get(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
+        int idx)
+{
+}
+
+/* The mode_config.mutex will be held for this call */
+static void armada_drm_crtc_dpms(struct drm_crtc *crtc, int dpms)
+{
+        struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+
+        if (dcrtc->dpms != dpms) {
+                dcrtc->dpms = dpms;
+                armada_drm_crtc_update(dcrtc);
+                if (dpms_blanked(dpms))
+                        armada_drm_vblank_off(dcrtc);
+        }
+}
+
+/*
+ * Prepare for a mode set.  Turn off overlay to ensure that we don't end
+ * up with the overlay size being bigger than the active screen size.
+ * We rely upon X refreshing this state after the mode set has completed.
+ *
+ * The mode_config.mutex will be held for this call
+ */
+static void armada_drm_crtc_prepare(struct drm_crtc *crtc)
+{
+        struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+        struct drm_plane *plane;
+
+        /*
+         * If we have an overlay plane associated with this CRTC, disable
+         * it before the modeset to avoid its coordinates being outside
+         * the new mode parameters.  DRM doesn't provide help with this.
+         */
+        plane = dcrtc->plane;
+        if (plane) {
+                struct drm_framebuffer *fb = plane->fb;
+
+                plane->funcs->disable_plane(plane);
+                plane->fb = NULL;
+                plane->crtc = NULL;
+                drm_framebuffer_unreference(fb);
+        }
+}
+
+/* The mode_config.mutex will be held for this call */
+static void armada_drm_crtc_commit(struct drm_crtc *crtc)
+{
+        struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+
+        if (dcrtc->dpms != DRM_MODE_DPMS_ON) {
+                dcrtc->dpms = DRM_MODE_DPMS_ON;
+                armada_drm_crtc_update(dcrtc);
+        }
+}
+
+/* The mode_config.mutex will be held for this call */
+static bool armada_drm_crtc_mode_fixup(struct drm_crtc *crtc,
+        const struct drm_display_mode *mode, struct drm_display_mode *adj)
+{
+        struct armada_private *priv = crtc->dev->dev_private;
+        struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+        int ret;
+
+        /* We can't do interlaced modes if we don't have the SPU_ADV_REG */
+        if (!priv->variant->has_spu_adv_reg &&
+            adj->flags & DRM_MODE_FLAG_INTERLACE)
+                return false;
+
+        /* Check whether the display mode is possible */
+        ret = priv->variant->crtc_compute_clock(dcrtc, adj, NULL);
+        if (ret)
+                return false;
+
+        return true;
+}
+
+void armada_drm_crtc_irq(struct armada_crtc *dcrtc, u32 stat)
+{
+        struct armada_vbl_event *e, *n;
+        void __iomem *base = dcrtc->base;
+
+        if (stat & DMA_FF_UNDERFLOW)
+                DRM_ERROR("video underflow on crtc %u\n", dcrtc->num);
+        if (stat & GRA_FF_UNDERFLOW)
+                DRM_ERROR("graphics underflow on crtc %u\n", dcrtc->num);
+
+        if (stat & VSYNC_IRQ)
+                drm_handle_vblank(dcrtc->crtc.dev, dcrtc->num);
+
+        spin_lock(&dcrtc->irq_lock);
+
+        list_for_each_entry_safe(e, n, &dcrtc->vbl_list, node) {
+                list_del_init(&e->node);
+                drm_vblank_put(dcrtc->crtc.dev, dcrtc->num);
+                e->fn(dcrtc, e->data);
+        }
+
+        if (stat & GRA_FRAME_IRQ && dcrtc->interlaced) {
+                int i = stat & GRA_FRAME_IRQ0 ? 0 : 1;
+                uint32_t val;
+
+                writel_relaxed(dcrtc->v[i].spu_v_porch, base + LCD_SPU_V_PORCH);
+                writel_relaxed(dcrtc->v[i].spu_v_h_total,
+                               base + LCD_SPUT_V_H_TOTAL);
+
+                val = readl_relaxed(base + LCD_SPU_ADV_REG);
+                val &= ~(ADV_VSYNC_L_OFF | ADV_VSYNC_H_OFF | ADV_VSYNCOFFEN);
+                val |= dcrtc->v[i].spu_adv_reg;
+                writel_relaxed(val, base + LCD_SPU_ADV_REG);
+        }
+
+        if (stat & DUMB_FRAMEDONE && dcrtc->cursor_update) {
+                writel_relaxed(dcrtc->cursor_hw_pos,
+                               base + LCD_SPU_HWC_OVSA_HPXL_VLN);
+                writel_relaxed(dcrtc->cursor_hw_sz,
+                               base + LCD_SPU_HWC_HPXL_VLN);
+                armada_updatel(CFG_HWC_ENA,
+                               CFG_HWC_ENA | CFG_HWC_1BITMOD | CFG_HWC_1BITENA,
+                               base + LCD_SPU_DMA_CTRL0);
+                dcrtc->cursor_update = false;
+                armada_drm_crtc_disable_irq(dcrtc, DUMB_FRAMEDONE_ENA);
+        }
+
+        spin_unlock(&dcrtc->irq_lock);
+
+        if (stat & GRA_FRAME_IRQ) {
+                struct drm_device *dev = dcrtc->crtc.dev;
+
+                spin_lock(&dev->event_lock);
+                if (dcrtc->frame_work)
+                        armada_drm_crtc_complete_frame_work(dcrtc);
+                spin_unlock(&dev->event_lock);
+
+                wake_up(&dcrtc->frame_wait);
+        }
+}
+
+/* These are locked by dev->vbl_lock */
+void armada_drm_crtc_disable_irq(struct armada_crtc *dcrtc, u32 mask)
+{
+        if (dcrtc->irq_ena & mask) {
+                dcrtc->irq_ena &= ~mask;
+                writel(dcrtc->irq_ena, dcrtc->base + LCD_SPU_IRQ_ENA);
+        }
+}
+
+void armada_drm_crtc_enable_irq(struct armada_crtc *dcrtc, u32 mask)
+{
+        if ((dcrtc->irq_ena & mask) != mask) {
+                dcrtc->irq_ena |= mask;
+                writel(dcrtc->irq_ena, dcrtc->base + LCD_SPU_IRQ_ENA);
+                if (readl_relaxed(dcrtc->base + LCD_SPU_IRQ_ISR) & mask)
+                        writel(0, dcrtc->base + LCD_SPU_IRQ_ISR);
+        }
+}
+
+static uint32_t armada_drm_crtc_calculate_csc(struct armada_crtc *dcrtc)
+{
+        struct drm_display_mode *adj = &dcrtc->crtc.mode;
+        uint32_t val = 0;
+
+        if (dcrtc->csc_yuv_mode == CSC_YUV_CCIR709)
+                val |= CFG_CSC_YUV_CCIR709;
+        if (dcrtc->csc_rgb_mode == CSC_RGB_STUDIO)
+                val |= CFG_CSC_RGB_STUDIO;
+
+        /*
+         * In auto mode, set the colorimetry, based upon the HDMI spec.
+         * 1280x720p, 1920x1080p and 1920x1080i use ITU709, others use
+         * ITU601.  It may be more appropriate to set this depending on
+         * the source - but what if the graphic frame is YUV and the
+         * video frame is RGB?
+         */
+        if ((adj->hdisplay == 1280 && adj->vdisplay == 720 &&
+             !(adj->flags & DRM_MODE_FLAG_INTERLACE)) ||
+            (adj->hdisplay == 1920 && adj->vdisplay == 1080)) {
+                if (dcrtc->csc_yuv_mode == CSC_AUTO)
+                        val |= CFG_CSC_YUV_CCIR709;
+        }
+
+        /*
+         * We assume we're connected to a TV-like device, so the YUV->RGB
+         * conversion should produce a limited range.  We should set this
+         * depending on the connectors attached to this CRTC, and what
+         * kind of device they report being connected.
+         */
+        if (dcrtc->csc_rgb_mode == CSC_AUTO)
+                val |= CFG_CSC_RGB_STUDIO;
+
+        return val;
+}
+
+/* The mode_config.mutex will be held for this call */
+static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
+        struct drm_display_mode *mode, struct drm_display_mode *adj,
+        int x, int y, struct drm_framebuffer *old_fb)
+{
+        struct armada_private *priv = crtc->dev->dev_private;
+        struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+        struct armada_regs regs[17];
+        uint32_t lm, rm, tm, bm, val, sclk;
+        unsigned long flags;
+        unsigned i;
+        bool interlaced;
+
+        drm_framebuffer_reference(crtc->fb);
+
+        interlaced = !!(adj->flags & DRM_MODE_FLAG_INTERLACE);
+
+        i = armada_drm_crtc_calc_fb(dcrtc->crtc.fb, x, y, regs, interlaced);
+
+        rm = adj->crtc_hsync_start - adj->crtc_hdisplay;
+        lm = adj->crtc_htotal - adj->crtc_hsync_end;
+        bm = adj->crtc_vsync_start - adj->crtc_vdisplay;
+        tm = adj->crtc_vtotal - adj->crtc_vsync_end;
+
+        DRM_DEBUG_DRIVER("H: %d %d %d %d lm %d rm %d\n",
+                adj->crtc_hdisplay,
+                adj->crtc_hsync_start,
+                adj->crtc_hsync_end,
+                adj->crtc_htotal, lm, rm);
+        DRM_DEBUG_DRIVER("V: %d %d %d %d tm %d bm %d\n",
+                adj->crtc_vdisplay,
+                adj->crtc_vsync_start,
+                adj->crtc_vsync_end,
+                adj->crtc_vtotal, tm, bm);
+
+        /* Wait for pending flips to complete */
+        wait_event(dcrtc->frame_wait, !dcrtc->frame_work);
+
+        drm_vblank_pre_modeset(crtc->dev, dcrtc->num);
+
+        crtc->mode = *adj;
+
+        val = dcrtc->dumb_ctrl & ~CFG_DUMB_ENA;
+        if (val != dcrtc->dumb_ctrl) {
+                dcrtc->dumb_ctrl = val;
+                writel_relaxed(val, dcrtc->base + LCD_SPU_DUMB_CTRL);
+        }
+
+        /* Now compute the divider for real */
+        priv->variant->crtc_compute_clock(dcrtc, adj, &sclk);
+
+        /* Ensure graphic fifo is enabled */
+        armada_reg_queue_mod(regs, i, 0, CFG_PDWN64x66, LCD_SPU_SRAM_PARA1);
+        armada_reg_queue_set(regs, i, sclk, LCD_CFG_SCLK_DIV);
+
+        if (interlaced ^ dcrtc->interlaced) {
+                if (adj->flags & DRM_MODE_FLAG_INTERLACE)
+                        drm_vblank_get(dcrtc->crtc.dev, dcrtc->num);
+                else
+                        drm_vblank_put(dcrtc->crtc.dev, dcrtc->num);
+                dcrtc->interlaced = interlaced;
+        }
+
+        spin_lock_irqsave(&dcrtc->irq_lock, flags);
+
+        /* Even interlaced/progressive frame */
+        dcrtc->v[1].spu_v_h_total = adj->crtc_vtotal << 16 |
+                                    adj->crtc_htotal;
+        dcrtc->v[1].spu_v_porch = tm << 16 | bm;
+        val = adj->crtc_hsync_start;
+        dcrtc->v[1].spu_adv_reg = val << 20 | val | ADV_VSYNCOFFEN |
+                priv->variant->spu_adv_reg;
+
+        if (interlaced) {
+                /* Odd interlaced frame */
+                dcrtc->v[0].spu_v_h_total = dcrtc->v[1].spu_v_h_total +
+                                                (1 << 16);
+                dcrtc->v[0].spu_v_porch = dcrtc->v[1].spu_v_porch + 1;
+                val = adj->crtc_hsync_start - adj->crtc_htotal / 2;
+                dcrtc->v[0].spu_adv_reg = val << 20 | val | ADV_VSYNCOFFEN |
+                        priv->variant->spu_adv_reg;
+        } else {
+                dcrtc->v[0] = dcrtc->v[1];
+        }
+
+        val = adj->crtc_vdisplay << 16 | adj->crtc_hdisplay;
+
+        armada_reg_queue_set(regs, i, val, LCD_SPU_V_H_ACTIVE);
+        armada_reg_queue_set(regs, i, val, LCD_SPU_GRA_HPXL_VLN);
+        armada_reg_queue_set(regs, i, val, LCD_SPU_GZM_HPXL_VLN);
+        armada_reg_queue_set(regs, i, (lm << 16) | rm, LCD_SPU_H_PORCH);
+        armada_reg_queue_set(regs, i, dcrtc->v[0].spu_v_porch, LCD_SPU_V_PORCH);
+        armada_reg_queue_set(regs, i, dcrtc->v[0].spu_v_h_total,
+                           LCD_SPUT_V_H_TOTAL);
+
+        if (priv->variant->has_spu_adv_reg) {
+                armada_reg_queue_mod(regs, i, dcrtc->v[0].spu_adv_reg,
+                                     ADV_VSYNC_L_OFF | ADV_VSYNC_H_OFF |
+                                     ADV_VSYNCOFFEN, LCD_SPU_ADV_REG);
+        }
+
+        val = CFG_GRA_ENA | CFG_GRA_HSMOOTH;
+        val |= CFG_GRA_FMT(drm_fb_to_armada_fb(dcrtc->crtc.fb)->fmt);
+        val |= CFG_GRA_MOD(drm_fb_to_armada_fb(dcrtc->crtc.fb)->mod);
+
+        if (drm_fb_to_armada_fb(dcrtc->crtc.fb)->fmt > CFG_420)
+                val |= CFG_PALETTE_ENA;
+
+        if (interlaced)
+                val |= CFG_GRA_FTOGGLE;
+
+        armada_reg_queue_mod(regs, i, val, CFG_GRAFORMAT |
+                             CFG_GRA_MOD(CFG_SWAPRB | CFG_SWAPUV |
+                                         CFG_SWAPYU | CFG_YUV2RGB) |
+                             CFG_PALETTE_ENA | CFG_GRA_FTOGGLE,
+                             LCD_SPU_DMA_CTRL0);
+
+        val = adj->flags & DRM_MODE_FLAG_NVSYNC ? CFG_VSYNC_INV : 0;
+        armada_reg_queue_mod(regs, i, val, CFG_VSYNC_INV, LCD_SPU_DMA_CTRL1);
+
+        val = dcrtc->spu_iopad_ctrl | armada_drm_crtc_calculate_csc(dcrtc);
+        armada_reg_queue_set(regs, i, val, LCD_SPU_IOPAD_CONTROL);
+        armada_reg_queue_end(regs, i);
+
+        armada_drm_crtc_update_regs(dcrtc, regs);
+        spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
+
+        armada_drm_crtc_update(dcrtc);
+
+        drm_vblank_post_modeset(crtc->dev, dcrtc->num);
+        armada_drm_crtc_finish_fb(dcrtc, old_fb, dpms_blanked(dcrtc->dpms));
+
+        return 0;
+}
+
+/* The mode_config.mutex will be held for this call */
+static int armada_drm_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
+        struct drm_framebuffer *old_fb)
+{
+        struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+        struct armada_regs regs[4];
+        unsigned i;
+
+        i = armada_drm_crtc_calc_fb(crtc->fb, crtc->x, crtc->y, regs,
+                                    dcrtc->interlaced);
+        armada_reg_queue_end(regs, i);
+
+        /* Wait for pending flips to complete */
+        wait_event(dcrtc->frame_wait, !dcrtc->frame_work);
+
+        /* Take a reference to the new fb as we're using it */
+        drm_framebuffer_reference(crtc->fb);
+
+        /* Update the base in the CRTC */
+        armada_drm_crtc_update_regs(dcrtc, regs);
+
+        /* Drop our previously held reference */
+        armada_drm_crtc_finish_fb(dcrtc, old_fb, dpms_blanked(dcrtc->dpms));
+
+        return 0;
+}
+
+static void armada_drm_crtc_load_lut(struct drm_crtc *crtc)
+{
+}
+
+/* The mode_config.mutex will be held for this call */
+static void armada_drm_crtc_disable(struct drm_crtc *crtc)
+{
+        struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+
+        armada_drm_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
+        armada_drm_crtc_finish_fb(dcrtc, crtc->fb, true);
+
+        /* Power down most RAMs and FIFOs */
+        writel_relaxed(CFG_PDWN256x32 | CFG_PDWN256x24 | CFG_PDWN256x8 |
+                       CFG_PDWN32x32 | CFG_PDWN16x66 | CFG_PDWN32x66 |
+                       CFG_PDWN64x66, dcrtc->base + LCD_SPU_SRAM_PARA1);
+}
+
+static const struct drm_crtc_helper_funcs armada_crtc_helper_funcs = {
+        .dpms           = armada_drm_crtc_dpms,
+        .prepare        = armada_drm_crtc_prepare,
+        .commit         = armada_drm_crtc_commit,
+        .mode_fixup     = armada_drm_crtc_mode_fixup,
+        .mode_set       = armada_drm_crtc_mode_set,
+        .mode_set_base  = armada_drm_crtc_mode_set_base,
+        .load_lut       = armada_drm_crtc_load_lut,
+        .disable        = armada_drm_crtc_disable,
+};
+
+static void armada_load_cursor_argb(void __iomem *base, uint32_t *pix,
+        unsigned stride, unsigned width, unsigned height)
+{
+        uint32_t addr;
+        unsigned y;
+
+        addr = SRAM_HWC32_RAM1;
+        for (y = 0; y < height; y++) {
+                uint32_t *p = &pix[y * stride];
+                unsigned x;
+
+                for (x = 0; x < width; x++, p++) {
+                        uint32_t val = *p;
+
+                        val = (val & 0xff00ff00) |
+                              (val & 0x000000ff) << 16 |
+                              (val & 0x00ff0000) >> 16;
+
+                        writel_relaxed(val,
+                                       base + LCD_SPU_SRAM_WRDAT);
+                        writel_relaxed(addr | SRAM_WRITE,
+                                       base + LCD_SPU_SRAM_CTRL);
+                        addr += 1;
+                        if ((addr & 0x00ff) == 0)
+                                addr += 0xf00;
+                        if ((addr & 0x30ff) == 0)
+                                addr = SRAM_HWC32_RAM2;
+                }
+        }
+}
+
+static void armada_drm_crtc_cursor_tran(void __iomem *base)
+{
+        unsigned addr;
+
+        for (addr = 0; addr < 256; addr++) {
+                /* write the default value */
+                writel_relaxed(0x55555555, base + LCD_SPU_SRAM_WRDAT);
+                writel_relaxed(addr | SRAM_WRITE | SRAM_HWC32_TRAN,
+                               base + LCD_SPU_SRAM_CTRL);
+        }
+}
+
+static int armada_drm_crtc_cursor_update(struct armada_crtc *dcrtc, bool reload)
+{
+        uint32_t xoff, xscr, w = dcrtc->cursor_w, s;
+        uint32_t yoff, yscr, h = dcrtc->cursor_h;
+        uint32_t para1;
+
+        /*
+         * Calculate the visible width and height of the cursor,
+         * screen position, and the position in the cursor bitmap.
+         */
+        if (dcrtc->cursor_x < 0) {
+                xoff = -dcrtc->cursor_x;
+                xscr = 0;
+                w -= min(xoff, w);
+        } else if (dcrtc->cursor_x + w > dcrtc->crtc.mode.hdisplay) {
+                xoff = 0;
+                xscr = dcrtc->cursor_x;
+                w = max_t(int, dcrtc->crtc.mode.hdisplay - dcrtc->cursor_x, 0);
+        } else {
+                xoff = 0;
+                xscr = dcrtc->cursor_x;
+        }
+
+        if (dcrtc->cursor_y < 0) {
+                yoff = -dcrtc->cursor_y;
+                yscr = 0;
+                h -= min(yoff, h);
+        } else if (dcrtc->cursor_y + h > dcrtc->crtc.mode.vdisplay) {
+                yoff = 0;
+                yscr = dcrtc->cursor_y;
+                h = max_t(int, dcrtc->crtc.mode.vdisplay - dcrtc->cursor_y, 0);
+        } else {
+                yoff = 0;
+                yscr = dcrtc->cursor_y;
+        }
+
+        /* On interlaced modes, the vertical cursor size must be halved */
+        s = dcrtc->cursor_w;
+        if (dcrtc->interlaced) {
+                s *= 2;
+                yscr /= 2;
+                h /= 2;
+        }
+
+        if (!dcrtc->cursor_obj || !h || !w) {
+                spin_lock_irq(&dcrtc->irq_lock);
+                armada_drm_crtc_disable_irq(dcrtc, DUMB_FRAMEDONE_ENA);
+                dcrtc->cursor_update = false;
+                armada_updatel(0, CFG_HWC_ENA, dcrtc->base + LCD_SPU_DMA_CTRL0);
+                spin_unlock_irq(&dcrtc->irq_lock);
+                return 0;
+        }
+
+        para1 = readl_relaxed(dcrtc->base + LCD_SPU_SRAM_PARA1);
+        armada_updatel(CFG_CSB_256x32, CFG_CSB_256x32 | CFG_PDWN256x32,
+                       dcrtc->base + LCD_SPU_SRAM_PARA1);
+
+        /*
+         * Initialize the transparency if the SRAM was powered down.
+         * We must also reload the cursor data as well.
+         */
+        if (!(para1 & CFG_CSB_256x32)) {
+                armada_drm_crtc_cursor_tran(dcrtc->base);
+                reload = true;
+        }
+
+        if (dcrtc->cursor_hw_sz != (h << 16 | w)) {
+                spin_lock_irq(&dcrtc->irq_lock);
+                armada_drm_crtc_disable_irq(dcrtc, DUMB_FRAMEDONE_ENA);
+                dcrtc->cursor_update = false;
+                armada_updatel(0, CFG_HWC_ENA, dcrtc->base + LCD_SPU_DMA_CTRL0);
+                spin_unlock_irq(&dcrtc->irq_lock);
+                reload = true;
+        }
+        if (reload) {
+                struct armada_gem_object *obj = dcrtc->cursor_obj;
+                uint32_t *pix;
+                /* Set the top-left corner of the cursor image */
+                pix = obj->addr;
+                pix += yoff * s + xoff;
+                armada_load_cursor_argb(dcrtc->base, pix, s, w, h);
+        }
+
+        /* Reload the cursor position, size and enable in the IRQ handler */
+        spin_lock_irq(&dcrtc->irq_lock);
+        dcrtc->cursor_hw_pos = yscr << 16 | xscr;
+        dcrtc->cursor_hw_sz = h << 16 | w;
+        dcrtc->cursor_update = true;
+        armada_drm_crtc_enable_irq(dcrtc, DUMB_FRAMEDONE_ENA);
+        spin_unlock_irq(&dcrtc->irq_lock);
+
+        return 0;
+}
+
+static void cursor_update(void *data)
+{
+        armada_drm_crtc_cursor_update(data, true);
+}
+
+static int armada_drm_crtc_cursor_set(struct drm_crtc *crtc,
+        struct drm_file *file, uint32_t handle, uint32_t w, uint32_t h)
+{
+        struct drm_device *dev = crtc->dev;
+        struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+        struct armada_private *priv = crtc->dev->dev_private;
+        struct armada_gem_object *obj = NULL;
+        int ret;
+
+        /* If no cursor support, replicate drm's return value */
+        if (!priv->variant->has_spu_adv_reg)
+                return -ENXIO;
+
+        if (handle && w > 0 && h > 0) {
+                /* maximum size is 64x32 or 32x64 */
+                if (w > 64 || h > 64 || (w > 32 && h > 32))
+                        return -ENOMEM;
+
+                obj = armada_gem_object_lookup(dev, file, handle);
+                if (!obj)
+                        return -ENOENT;
+
+                /* Must be a kernel-mapped object */
+                if (!obj->addr) {
+                        drm_gem_object_unreference_unlocked(&obj->obj);
+                        return -EINVAL;
+                }
+
+                if (obj->obj.size < w * h * 4) {
+                        DRM_ERROR("buffer is too small\n");
+                        drm_gem_object_unreference_unlocked(&obj->obj);
+                        return -ENOMEM;
+                }
+        }
+
+        mutex_lock(&dev->struct_mutex);
+        if (dcrtc->cursor_obj) {
+                dcrtc->cursor_obj->update = NULL;
+                dcrtc->cursor_obj->update_data = NULL;
+                drm_gem_object_unreference(&dcrtc->cursor_obj->obj);
+        }
+        dcrtc->cursor_obj = obj;
+        dcrtc->cursor_w = w;
+        dcrtc->cursor_h = h;
+        ret = armada_drm_crtc_cursor_update(dcrtc, true);
+        if (obj) {
+                obj->update_data = dcrtc;
+                obj->update = cursor_update;
+        }
+        mutex_unlock(&dev->struct_mutex);
+
+        return ret;
+}
+
+static int armada_drm_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
+{
+        struct drm_device *dev = crtc->dev;
+        struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+        struct armada_private *priv = crtc->dev->dev_private;
+        int ret;
+
+        /* If no cursor support, replicate drm's return value */
+        if (!priv->variant->has_spu_adv_reg)
+                return -EFAULT;
+
+        mutex_lock(&dev->struct_mutex);
+        dcrtc->cursor_x = x;
+        dcrtc->cursor_y = y;
+        ret = armada_drm_crtc_cursor_update(dcrtc, false);
+        mutex_unlock(&dev->struct_mutex);
+
+        return ret;
+}
+
+static void armada_drm_crtc_destroy(struct drm_crtc *crtc)
+{
+        struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+        struct armada_private *priv = crtc->dev->dev_private;
+
+        if (dcrtc->cursor_obj)
+                drm_gem_object_unreference(&dcrtc->cursor_obj->obj);
+
+        priv->dcrtc[dcrtc->num] = NULL;
+        drm_crtc_cleanup(&dcrtc->crtc);
+
+        if (!IS_ERR(dcrtc->clk))
+                clk_disable_unprepare(dcrtc->clk);
+
+        kfree(dcrtc);
+}
+
+/*
+ * The mode_config lock is held here, to prevent races between this
+ * and a mode_set.
+ */
+static int armada_drm_crtc_page_flip(struct drm_crtc *crtc,
+        struct drm_framebuffer *fb, struct drm_pending_vblank_event *event, uint32_t page_flip_flags)
+{
+        struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+        struct armada_frame_work *work;
+        struct drm_device *dev = crtc->dev;
+        unsigned long flags;
+        unsigned i;
+        int ret;
+
+        /* We don't support changing the pixel format */
+        if (fb->pixel_format != crtc->fb->pixel_format)
+                return -EINVAL;
+
+        work = kmalloc(sizeof(*work), GFP_KERNEL);
+        if (!work)
+                return -ENOMEM;
+
+        work->event = event;
+        work->old_fb = dcrtc->crtc.fb;
+
+        i = armada_drm_crtc_calc_fb(fb, crtc->x, crtc->y, work->regs,
+                                    dcrtc->interlaced);
+        armada_reg_queue_end(work->regs, i);
+
+        /*
+         * Hold the old framebuffer for the work - DRM appears to drop our
+         * reference to the old framebuffer in drm_mode_page_flip_ioctl().
+         */
+        drm_framebuffer_reference(work->old_fb);
+
+        ret = armada_drm_crtc_queue_frame_work(dcrtc, work);
+        if (ret) {
+                /*
+                 * Undo our reference above; DRM does not drop the reference
+                 * to this object on error, so that's okay.
+                 */
+                drm_framebuffer_unreference(work->old_fb);
+                kfree(work);
+                return ret;
+        }
+
+        /*
+         * Don't take a reference on the new framebuffer;
+         * drm_mode_page_flip_ioctl() has already grabbed a reference and
+         * will _not_ drop that reference on successful return from this
+         * function.  Simply mark this new framebuffer as the current one.
+         */
+        dcrtc->crtc.fb = fb;
+
+        /*
+         * Finally, if the display is blanked, we won't receive an
+         * interrupt, so complete it now.
+         */
+        if (dpms_blanked(dcrtc->dpms)) {
+                spin_lock_irqsave(&dev->event_lock, flags);
+                if (dcrtc->frame_work)
+                        armada_drm_crtc_complete_frame_work(dcrtc);
+                spin_unlock_irqrestore(&dev->event_lock, flags);
+        }
+
+        return 0;
+}
+
+static int
+armada_drm_crtc_set_property(struct drm_crtc *crtc,
+        struct drm_property *property, uint64_t val)
+{
+        struct armada_private *priv = crtc->dev->dev_private;
+        struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+        bool update_csc = false;
+
+        if (property == priv->csc_yuv_prop) {
+                dcrtc->csc_yuv_mode = val;
+                update_csc = true;
+        } else if (property == priv->csc_rgb_prop) {
+                dcrtc->csc_rgb_mode = val;
+                update_csc = true;
+        }
+
+        if (update_csc) {
+                uint32_t val;
+
+                val = dcrtc->spu_iopad_ctrl |
+                      armada_drm_crtc_calculate_csc(dcrtc);
+                writel_relaxed(val, dcrtc->base + LCD_SPU_IOPAD_CONTROL);
+        }
+
+        return 0;
+}
+
+static struct drm_crtc_funcs armada_crtc_funcs = {
+        .cursor_set     = armada_drm_crtc_cursor_set,
+        .cursor_move    = armada_drm_crtc_cursor_move,
+        .destroy        = armada_drm_crtc_destroy,
+        .set_config     = drm_crtc_helper_set_config,
+        .page_flip      = armada_drm_crtc_page_flip,
+        .set_property   = armada_drm_crtc_set_property,
+};
+
+static struct drm_prop_enum_list armada_drm_csc_yuv_enum_list[] = {
+        { CSC_AUTO,        "Auto" },
+        { CSC_YUV_CCIR601, "CCIR601" },
+        { CSC_YUV_CCIR709, "CCIR709" },
+};
+
+static struct drm_prop_enum_list armada_drm_csc_rgb_enum_list[] = {
+        { CSC_AUTO,         "Auto" },
+        { CSC_RGB_COMPUTER, "Computer system" },
+        { CSC_RGB_STUDIO,   "Studio" },
+};
+
+static int armada_drm_crtc_create_properties(struct drm_device *dev)
+{
+        struct armada_private *priv = dev->dev_private;
+
+        if (priv->csc_yuv_prop)
+                return 0;
+
+        priv->csc_yuv_prop = drm_property_create_enum(dev, 0,
+                                "CSC_YUV", armada_drm_csc_yuv_enum_list,
+                                ARRAY_SIZE(armada_drm_csc_yuv_enum_list));
+        priv->csc_rgb_prop = drm_property_create_enum(dev, 0,
+                                "CSC_RGB", armada_drm_csc_rgb_enum_list,
+                                ARRAY_SIZE(armada_drm_csc_rgb_enum_list));
+
+        if (!priv->csc_yuv_prop || !priv->csc_rgb_prop)
+                return -ENOMEM;
+
+        return 0;
+}
+
+int armada_drm_crtc_create(struct drm_device *dev, unsigned num,
+        struct resource *res)
+{
+        struct armada_private *priv = dev->dev_private;
+        struct armada_crtc *dcrtc;
+        void __iomem *base;
+        int ret;
+
+        ret = armada_drm_crtc_create_properties(dev);
+        if (ret)
+                return ret;
+
+        base = devm_request_and_ioremap(dev->dev, res);
+        if (!base) {
+                DRM_ERROR("failed to ioremap register\n");
+                return -ENOMEM;
+        }
+
+        dcrtc = kzalloc(sizeof(*dcrtc), GFP_KERNEL);
+        if (!dcrtc) {
+                DRM_ERROR("failed to allocate Armada crtc\n");
+                return -ENOMEM;
+        }
+
+        dcrtc->base = base;
+        dcrtc->num = num;
+        dcrtc->clk = ERR_PTR(-EINVAL);
+        dcrtc->csc_yuv_mode = CSC_AUTO;
+        dcrtc->csc_rgb_mode = CSC_AUTO;
+        dcrtc->cfg_dumb_ctrl = DUMB24_RGB888_0;
+        dcrtc->spu_iopad_ctrl = CFG_VSCALE_LN_EN | CFG_IOPAD_DUMB24;
+        spin_lock_init(&dcrtc->irq_lock);
+        dcrtc->irq_ena = CLEAN_SPU_IRQ_ISR;
+        INIT_LIST_HEAD(&dcrtc->vbl_list);
+        init_waitqueue_head(&dcrtc->frame_wait);
+
+        /* Initialize some registers which we don't otherwise set */
+        writel_relaxed(0x00000001, dcrtc->base + LCD_CFG_SCLK_DIV);
+        writel_relaxed(0x00000000, dcrtc->base + LCD_SPU_BLANKCOLOR);
+        writel_relaxed(dcrtc->spu_iopad_ctrl,
+                       dcrtc->base + LCD_SPU_IOPAD_CONTROL);
+        writel_relaxed(0x00000000, dcrtc->base + LCD_SPU_SRAM_PARA0);
+        writel_relaxed(CFG_PDWN256x32 | CFG_PDWN256x24 | CFG_PDWN256x8 |
+                       CFG_PDWN32x32 | CFG_PDWN16x66 | CFG_PDWN32x66 |
+                       CFG_PDWN64x66, dcrtc->base + LCD_SPU_SRAM_PARA1);
+        writel_relaxed(0x2032ff81, dcrtc->base + LCD_SPU_DMA_CTRL1);
+        writel_relaxed(0x00000000, dcrtc->base + LCD_SPU_GRA_OVSA_HPXL_VLN);
+
+        if (priv->variant->crtc_init) {
+                ret = priv->variant->crtc_init(dcrtc);
+                if (ret) {
+                        kfree(dcrtc);
+                        return ret;
+                }
+        }
+
+        /* Ensure AXI pipeline is enabled */
+        armada_updatel(CFG_ARBFAST_ENA, 0, dcrtc->base + LCD_SPU_DMA_CTRL0);
+
+        priv->dcrtc[dcrtc->num] = dcrtc;
+
+        drm_crtc_init(dev, &dcrtc->crtc, &armada_crtc_funcs);
+        drm_crtc_helper_add(&dcrtc->crtc, &armada_crtc_helper_funcs);
+
+        drm_object_attach_property(&dcrtc->crtc.base, priv->csc_yuv_prop,
+                                   dcrtc->csc_yuv_mode);
+        drm_object_attach_property(&dcrtc->crtc.base, priv->csc_rgb_prop,
+                                   dcrtc->csc_rgb_mode);
+
+        return armada_overlay_plane_create(dev, 1 << dcrtc->num);
+}
diff --git a/drivers/gpu/drm/armada/armada_crtc.h b/drivers/gpu/drm/armada/armada_crtc.h
new file mode 100644
index 000000000000..9c10a07e7492
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_crtc.h
@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef ARMADA_CRTC_H
+#define ARMADA_CRTC_H
+
+struct armada_gem_object;
+
+struct armada_regs {
+        uint32_t offset;
+        uint32_t mask;
+        uint32_t val;
+};
+
+#define armada_reg_queue_mod(_r, _i, _v, _m, _o)        \
+        do {                                    \
+                struct armada_regs *__reg = _r; \
+                __reg[_i].offset = _o;          \
+                __reg[_i].mask = ~(_m);         \
+                __reg[_i].val = _v;             \
+                _i++;                           \
+        } while (0)
+
+#define armada_reg_queue_set(_r, _i, _v, _o)    \
+        armada_reg_queue_mod(_r, _i, _v, ~0, _o)
+
+#define armada_reg_queue_end(_r, _i)            \
+        armada_reg_queue_mod(_r, _i, 0, 0, ~0)
+
+struct armada_frame_work;
+
+struct armada_crtc {
+        struct drm_crtc         crtc;
+        unsigned                num;
+        void __iomem            *base;
+        struct clk              *clk;
+        struct {
+                uint32_t        spu_v_h_total;
+                uint32_t        spu_v_porch;
+                uint32_t        spu_adv_reg;
+        } v[2];
+        bool                    interlaced;
+        bool                    cursor_update;
+        uint8_t                 csc_yuv_mode;
+        uint8_t                 csc_rgb_mode;
+
+        struct drm_plane        *plane;
+
+        struct armada_gem_object        *cursor_obj;
+        int                     cursor_x;
+        int                     cursor_y;
+        uint32_t                cursor_hw_pos;
+        uint32_t                cursor_hw_sz;
+        uint32_t                cursor_w;
+        uint32_t                cursor_h;
+
+        int                     dpms;
+        uint32_t                cfg_dumb_ctrl;
+        uint32_t                dumb_ctrl;
+        uint32_t                spu_iopad_ctrl;
+
+        wait_queue_head_t       frame_wait;
+        struct armada_frame_work *frame_work;
+
+        spinlock_t              irq_lock;
+        uint32_t                irq_ena;
+        struct list_head        vbl_list;
+};
+#define drm_to_armada_crtc(c) container_of(c, struct armada_crtc, crtc)
+
+int armada_drm_crtc_create(struct drm_device *, unsigned, struct resource *);
+void armada_drm_crtc_gamma_set(struct drm_crtc *, u16, u16, u16, int);
+void armada_drm_crtc_gamma_get(struct drm_crtc *, u16 *, u16 *, u16 *, int);
+void armada_drm_crtc_irq(struct armada_crtc *, u32);
+void armada_drm_crtc_disable_irq(struct armada_crtc *, u32);
+void armada_drm_crtc_enable_irq(struct armada_crtc *, u32);
+void armada_drm_crtc_update_regs(struct armada_crtc *, struct armada_regs *);
+
+#endif
diff --git a/drivers/gpu/drm/armada/armada_debugfs.c b/drivers/gpu/drm/armada/armada_debugfs.c
new file mode 100644
index 000000000000..471e45627f1e
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_debugfs.c
@@ -0,0 +1,177 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *  Rewritten from the dovefb driver, and Armada510 manuals.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/ctype.h>
+#include <linux/debugfs.h>
+#include <linux/module.h>
+#include <linux/seq_file.h>
+#include <drm/drmP.h>
+#include "armada_crtc.h"
+#include "armada_drm.h"
+
+static int armada_debugfs_gem_linear_show(struct seq_file *m, void *data)
+{
+        struct drm_info_node *node = m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct armada_private *priv = dev->dev_private;
+        int ret;
+
+        mutex_lock(&dev->struct_mutex);
+        ret = drm_mm_dump_table(m, &priv->linear);
+        mutex_unlock(&dev->struct_mutex);
+
+        return ret;
+}
+
+static int armada_debugfs_reg_show(struct seq_file *m, void *data)
+{
+        struct drm_device *dev = m->private;
+        struct armada_private *priv = dev->dev_private;
+        int n, i;
+
+        if (priv) {
+                for (n = 0; n < ARRAY_SIZE(priv->dcrtc); n++) {
+                        struct armada_crtc *dcrtc = priv->dcrtc[n];
+                        if (!dcrtc)
+                                continue;
+
+                        for (i = 0x84; i <= 0x1c4; i += 4) {
+                                uint32_t v = readl_relaxed(dcrtc->base + i);
+                                seq_printf(m, "%u: 0x%04x: 0x%08x\n", n, i, v);
+                        }
+                }
+        }
+
+        return 0;
+}
+
+static int armada_debugfs_reg_r_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, armada_debugfs_reg_show, inode->i_private);
+}
+
+static const struct file_operations fops_reg_r = {
+        .owner = THIS_MODULE,
+        .open = armada_debugfs_reg_r_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+};
+
+static int armada_debugfs_write(struct file *file, const char __user *ptr,
+        size_t len, loff_t *off)
+{
+        struct drm_device *dev = file->private_data;
+        struct armada_private *priv = dev->dev_private;
+        struct armada_crtc *dcrtc = priv->dcrtc[0];
+        char buf[32], *p;
+        uint32_t reg, val;
+        int ret;
+
+        if (*off != 0)
+                return 0;
+
+        if (len > sizeof(buf) - 1)
+                len = sizeof(buf) - 1;
+
+        ret = strncpy_from_user(buf, ptr, len);
+        if (ret < 0)
+                return ret;
+        buf[len] = '\0';
+
+        reg = simple_strtoul(buf, &p, 16);
+        if (!isspace(*p))
+                return -EINVAL;
+        val = simple_strtoul(p + 1, NULL, 16);
+
+        if (reg >= 0x84 && reg <= 0x1c4)
+                writel(val, dcrtc->base + reg);
+
+        return len;
+}
+
+static const struct file_operations fops_reg_w = {
+        .owner = THIS_MODULE,
+        .open = simple_open,
+        .write = armada_debugfs_write,
+        .llseek = noop_llseek,
+};
+
+static struct drm_info_list armada_debugfs_list[] = {
+        { "gem_linear", armada_debugfs_gem_linear_show, 0 },
+};
+#define ARMADA_DEBUGFS_ENTRIES ARRAY_SIZE(armada_debugfs_list)
+
+static int drm_add_fake_info_node(struct drm_minor *minor, struct dentry *ent,
+        const void *key)
+{
+        struct drm_info_node *node;
+
+        node = kmalloc(sizeof(struct drm_info_node), GFP_KERNEL);
+        if (node == NULL) {
+                debugfs_remove(ent);
+                return -ENOMEM;
+        }
+
+        node->minor = minor;
+        node->dent = ent;
+        node->info_ent = (void *) key;
+
+        mutex_lock(&minor->debugfs_lock);
+        list_add(&node->list, &minor->debugfs_list);
+        mutex_unlock(&minor->debugfs_lock);
+
+        return 0;
+}
+
+static int armada_debugfs_create(struct dentry *root, struct drm_minor *minor,
+        const char *name, umode_t mode, const struct file_operations *fops)
+{
+        struct dentry *de;
+
+        de = debugfs_create_file(name, mode, root, minor->dev, fops);
+
+        return drm_add_fake_info_node(minor, de, fops);
+}
+
+int armada_drm_debugfs_init(struct drm_minor *minor)
+{
+        int ret;
+
+        ret = drm_debugfs_create_files(armada_debugfs_list,
+                                       ARMADA_DEBUGFS_ENTRIES,
+                                       minor->debugfs_root, minor);
+        if (ret)
+                return ret;
+
+        ret = armada_debugfs_create(minor->debugfs_root, minor,
+                                   "reg", S_IFREG | S_IRUSR, &fops_reg_r);
+        if (ret)
+                goto err_1;
+
+        ret = armada_debugfs_create(minor->debugfs_root, minor,
+                                "reg_wr", S_IFREG | S_IWUSR, &fops_reg_w);
+        if (ret)
+                goto err_2;
+        return ret;
+
+ err_2:
+        drm_debugfs_remove_files((struct drm_info_list *)&fops_reg_r, 1, minor);
+ err_1:
+        drm_debugfs_remove_files(armada_debugfs_list, ARMADA_DEBUGFS_ENTRIES,
+                                 minor);
+        return ret;
+}
+
+void armada_drm_debugfs_cleanup(struct drm_minor *minor)
+{
+        drm_debugfs_remove_files((struct drm_info_list *)&fops_reg_w, 1, minor);
+        drm_debugfs_remove_files((struct drm_info_list *)&fops_reg_r, 1, minor);
+        drm_debugfs_remove_files(armada_debugfs_list, ARMADA_DEBUGFS_ENTRIES,
+                                 minor);
+}
diff --git a/drivers/gpu/drm/armada/armada_drm.h b/drivers/gpu/drm/armada/armada_drm.h
new file mode 100644
index 000000000000..eef09ec9a5ff
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_drm.h
@@ -0,0 +1,113 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef ARMADA_DRM_H
+#define ARMADA_DRM_H
+
+#include <linux/kfifo.h>
+#include <linux/io.h>
+#include <linux/workqueue.h>
+#include <drm/drmP.h>
+
+struct armada_crtc;
+struct armada_gem_object;
+struct clk;
+struct drm_fb_helper;
+
+static inline void
+armada_updatel(uint32_t val, uint32_t mask, void __iomem *ptr)
+{
+        uint32_t ov, v;
+
+        ov = v = readl_relaxed(ptr);
+        v = (v & ~mask) | val;
+        if (ov != v)
+                writel_relaxed(v, ptr);
+}
+
+static inline uint32_t armada_pitch(uint32_t width, uint32_t bpp)
+{
+        uint32_t pitch = bpp != 4 ? width * ((bpp + 7) / 8) : width / 2;
+
+        /* 88AP510 spec recommends pitch be a multiple of 128 */
+        return ALIGN(pitch, 128);
+}
+
+struct armada_vbl_event {
+        struct list_head        node;
+        void                    *data;
+        void                    (*fn)(struct armada_crtc *, void *);
+};
+void armada_drm_vbl_event_add(struct armada_crtc *,
+        struct armada_vbl_event *);
+void armada_drm_vbl_event_remove(struct armada_crtc *,
+        struct armada_vbl_event *);
+void armada_drm_vbl_event_remove_unlocked(struct armada_crtc *,
+        struct armada_vbl_event *);
+#define armada_drm_vbl_event_init(_e, _f, _d) do {      \
+        struct armada_vbl_event *__e = _e;              \
+        INIT_LIST_HEAD(&__e->node);                     \
+        __e->data = _d;                                 \
+        __e->fn = _f;                                   \
+} while (0)
+
+
+struct armada_private;
+
+struct armada_variant {
+        bool    has_spu_adv_reg;
+        uint32_t spu_adv_reg;
+        int (*init)(struct armada_private *, struct device *);
+        int (*crtc_init)(struct armada_crtc *);
+        int (*crtc_compute_clock)(struct armada_crtc *,
+                                  const struct drm_display_mode *,
+                                  uint32_t *);
+};
+
+/* Variant ops */
+extern const struct armada_variant armada510_ops;
+
+struct armada_private {
+        const struct armada_variant *variant;
+        struct work_struct      fb_unref_work;
+        DECLARE_KFIFO(fb_unref, struct drm_framebuffer *, 8);
+        struct drm_fb_helper    *fbdev;
+        struct armada_crtc      *dcrtc[2];
+        struct drm_mm           linear;
+        struct clk              *extclk[2];
+        struct drm_property     *csc_yuv_prop;
+        struct drm_property     *csc_rgb_prop;
+        struct drm_property     *colorkey_prop;
+        struct drm_property     *colorkey_min_prop;
+        struct drm_property     *colorkey_max_prop;
+        struct drm_property     *colorkey_val_prop;
+        struct drm_property     *colorkey_alpha_prop;
+        struct drm_property     *colorkey_mode_prop;
+        struct drm_property     *brightness_prop;
+        struct drm_property     *contrast_prop;
+        struct drm_property     *saturation_prop;
+#ifdef CONFIG_DEBUG_FS
+        struct dentry           *de;
+#endif
+};
+
+void __armada_drm_queue_unref_work(struct drm_device *,
+        struct drm_framebuffer *);
+void armada_drm_queue_unref_work(struct drm_device *,
+        struct drm_framebuffer *);
+
+extern const struct drm_mode_config_funcs armada_drm_mode_config_funcs;
+
+int armada_fbdev_init(struct drm_device *);
+void armada_fbdev_fini(struct drm_device *);
+
+int armada_overlay_plane_create(struct drm_device *, unsigned long);
+
+int armada_drm_debugfs_init(struct drm_minor *);
+void armada_drm_debugfs_cleanup(struct drm_minor *);
+
+#endif
diff --git a/drivers/gpu/drm/armada/armada_drv.c b/drivers/gpu/drm/armada/armada_drv.c
new file mode 100644
index 000000000000..4f2b28354915
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_drv.c
@@ -0,0 +1,421 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/clk.h>
+#include <linux/module.h>
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include "armada_crtc.h"
+#include "armada_drm.h"
+#include "armada_gem.h"
+#include "armada_hw.h"
+#include <drm/armada_drm.h>
+#include "armada_ioctlP.h"
+
+#ifdef CONFIG_DRM_ARMADA_TDA1998X
+#include <drm/i2c/tda998x.h>
+#include "armada_slave.h"
+
+static struct tda998x_encoder_params params = {
+        /* With 0x24, there is no translation between vp_out and int_vp
+        FB      LCD out Pins    VIP     Int Vp
+        R:23:16 R:7:0   VPC7:0  7:0     7:0[R]
+        G:15:8  G:15:8  VPB7:0  23:16   23:16[G]
+        B:7:0   B:23:16 VPA7:0  15:8    15:8[B]
+        */
+        .swap_a = 2,
+        .swap_b = 3,
+        .swap_c = 4,
+        .swap_d = 5,
+        .swap_e = 0,
+        .swap_f = 1,
+        .audio_cfg = BIT(2),
+        .audio_frame[1] = 1,
+        .audio_format = AFMT_SPDIF,
+        .audio_sample_rate = 44100,
+};
+
+static const struct armada_drm_slave_config tda19988_config = {
+        .i2c_adapter_id = 0,
+        .crtcs = 1 << 0, /* Only LCD0 at the moment */
+        .polled = DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT,
+        .interlace_allowed = true,
+        .info = {
+                .type = "tda998x",
+                .addr = 0x70,
+                .platform_data = &params,
+        },
+};
+#endif
+
+static void armada_drm_unref_work(struct work_struct *work)
+{
+        struct armada_private *priv =
+                container_of(work, struct armada_private, fb_unref_work);
+        struct drm_framebuffer *fb;
+
+        while (kfifo_get(&priv->fb_unref, &fb))
+                drm_framebuffer_unreference(fb);
+}
+
+/* Must be called with dev->event_lock held */
+void __armada_drm_queue_unref_work(struct drm_device *dev,
+        struct drm_framebuffer *fb)
+{
+        struct armada_private *priv = dev->dev_private;
+
+        /*
+         * Yes, we really must jump through these hoops just to store a
+         * _pointer_ to something into the kfifo.  This is utterly insane
+         * and idiotic, because it kfifo requires the _data_ pointed to by
+         * the pointer const, not the pointer itself.  Not only that, but
+         * you have to pass a pointer _to_ the pointer you want stored.
+         */
+        const struct drm_framebuffer *silly_api_alert = fb;
+        WARN_ON(!kfifo_put(&priv->fb_unref, &silly_api_alert));
+        schedule_work(&priv->fb_unref_work);
+}
+
+void armada_drm_queue_unref_work(struct drm_device *dev,
+        struct drm_framebuffer *fb)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev->event_lock, flags);
+        __armada_drm_queue_unref_work(dev, fb);
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+}
+
+static int armada_drm_load(struct drm_device *dev, unsigned long flags)
+{
+        const struct platform_device_id *id;
+        struct armada_private *priv;
+        struct resource *res[ARRAY_SIZE(priv->dcrtc)];
+        struct resource *mem = NULL;
+        int ret, n, i;
+
+        memset(res, 0, sizeof(res));
+
+        for (n = i = 0; ; n++) {
+                struct resource *r = platform_get_resource(dev->platformdev,
+                                                           IORESOURCE_MEM, n);
+                if (!r)
+                        break;
+
+                /* Resources above 64K are graphics memory */
+                if (resource_size(r) > SZ_64K)
+                        mem = r;
+                else if (i < ARRAY_SIZE(priv->dcrtc))
+                        res[i++] = r;
+                else
+                        return -EINVAL;
+        }
+
+        if (!res[0] || !mem)
+                return -ENXIO;
+
+        if (!devm_request_mem_region(dev->dev, mem->start,
+                        resource_size(mem), "armada-drm"))
+                return -EBUSY;
+
+        priv = devm_kzalloc(dev->dev, sizeof(*priv), GFP_KERNEL);
+        if (!priv) {
+                DRM_ERROR("failed to allocate private\n");
+                return -ENOMEM;
+        }
+
+        dev->dev_private = priv;
+
+        /* Get the implementation specific driver data. */
+        id = platform_get_device_id(dev->platformdev);
+        if (!id)
+                return -ENXIO;
+
+        priv->variant = (struct armada_variant *)id->driver_data;
+
+        ret = priv->variant->init(priv, dev->dev);
+        if (ret)
+                return ret;
+
+        INIT_WORK(&priv->fb_unref_work, armada_drm_unref_work);
+        INIT_KFIFO(priv->fb_unref);
+
+        /* Mode setting support */
+        drm_mode_config_init(dev);
+        dev->mode_config.min_width = 320;
+        dev->mode_config.min_height = 200;
+
+        /*
+         * With vscale enabled, the maximum width is 1920 due to the
+         * 1920 by 3 lines RAM
+         */
+        dev->mode_config.max_width = 1920;
+        dev->mode_config.max_height = 2048;
+
+        dev->mode_config.preferred_depth = 24;
+        dev->mode_config.funcs = &armada_drm_mode_config_funcs;
+        drm_mm_init(&priv->linear, mem->start, resource_size(mem));
+
+        /* Create all LCD controllers */
+        for (n = 0; n < ARRAY_SIZE(priv->dcrtc); n++) {
+                if (!res[n])
+                        break;
+
+                ret = armada_drm_crtc_create(dev, n, res[n]);
+                if (ret)
+                        goto err_kms;
+        }
+
+#ifdef CONFIG_DRM_ARMADA_TDA1998X
+        ret = armada_drm_connector_slave_create(dev, &tda19988_config);
+        if (ret)
+                goto err_kms;
+#endif
+
+        ret = drm_vblank_init(dev, n);
+        if (ret)
+                goto err_kms;
+
+        ret = drm_irq_install(dev);
+        if (ret)
+                goto err_kms;
+
+        dev->vblank_disable_allowed = 1;
+
+        ret = armada_fbdev_init(dev);
+        if (ret)
+                goto err_irq;
+
+        drm_kms_helper_poll_init(dev);
+
+        return 0;
+
+ err_irq:
+        drm_irq_uninstall(dev);
+ err_kms:
+        drm_mode_config_cleanup(dev);
+        drm_mm_takedown(&priv->linear);
+        flush_work(&priv->fb_unref_work);
+
+        return ret;
+}
+
+static int armada_drm_unload(struct drm_device *dev)
+{
+        struct armada_private *priv = dev->dev_private;
+
+        drm_kms_helper_poll_fini(dev);
+        armada_fbdev_fini(dev);
+        drm_irq_uninstall(dev);
+        drm_mode_config_cleanup(dev);
+        drm_mm_takedown(&priv->linear);
+        flush_work(&priv->fb_unref_work);
+        dev->dev_private = NULL;
+
+        return 0;
+}
+
+void armada_drm_vbl_event_add(struct armada_crtc *dcrtc,
+        struct armada_vbl_event *evt)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&dcrtc->irq_lock, flags);
+        if (list_empty(&evt->node)) {
+                list_add_tail(&evt->node, &dcrtc->vbl_list);
+
+                drm_vblank_get(dcrtc->crtc.dev, dcrtc->num);
+        }
+        spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
+}
+
+void armada_drm_vbl_event_remove(struct armada_crtc *dcrtc,
+        struct armada_vbl_event *evt)
+{
+        if (!list_empty(&evt->node)) {
+                list_del_init(&evt->node);
+                drm_vblank_put(dcrtc->crtc.dev, dcrtc->num);
+        }
+}
+
+void armada_drm_vbl_event_remove_unlocked(struct armada_crtc *dcrtc,
+        struct armada_vbl_event *evt)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&dcrtc->irq_lock, flags);
+        armada_drm_vbl_event_remove(dcrtc, evt);
+        spin_unlock_irqrestore(&dcrtc->irq_lock, flags);
+}
+
+/* These are called under the vbl_lock. */
+static int armada_drm_enable_vblank(struct drm_device *dev, int crtc)
+{
+        struct armada_private *priv = dev->dev_private;
+        armada_drm_crtc_enable_irq(priv->dcrtc[crtc], VSYNC_IRQ_ENA);
+        return 0;
+}
+
+static void armada_drm_disable_vblank(struct drm_device *dev, int crtc)
+{
+        struct armada_private *priv = dev->dev_private;
+        armada_drm_crtc_disable_irq(priv->dcrtc[crtc], VSYNC_IRQ_ENA);
+}
+
+static irqreturn_t armada_drm_irq_handler(int irq, void *arg)
+{
+        struct drm_device *dev = arg;
+        struct armada_private *priv = dev->dev_private;
+        struct armada_crtc *dcrtc = priv->dcrtc[0];
+        uint32_t v, stat = readl_relaxed(dcrtc->base + LCD_SPU_IRQ_ISR);
+        irqreturn_t handled = IRQ_NONE;
+
+        /*
+         * This is rediculous - rather than writing bits to clear, we
+         * have to set the actual status register value.  This is racy.
+         */
+        writel_relaxed(0, dcrtc->base + LCD_SPU_IRQ_ISR);
+
+        /* Mask out those interrupts we haven't enabled */
+        v = stat & dcrtc->irq_ena;
+
+        if (v & (VSYNC_IRQ|GRA_FRAME_IRQ|DUMB_FRAMEDONE)) {
+                armada_drm_crtc_irq(dcrtc, stat);
+                handled = IRQ_HANDLED;
+        }
+
+        return handled;
+}
+
+static int armada_drm_irq_postinstall(struct drm_device *dev)
+{
+        struct armada_private *priv = dev->dev_private;
+        struct armada_crtc *dcrtc = priv->dcrtc[0];
+
+        spin_lock_irq(&dev->vbl_lock);
+        writel_relaxed(dcrtc->irq_ena, dcrtc->base + LCD_SPU_IRQ_ENA);
+        writel(0, dcrtc->base + LCD_SPU_IRQ_ISR);
+        spin_unlock_irq(&dev->vbl_lock);
+
+        return 0;
+}
+
+static void armada_drm_irq_uninstall(struct drm_device *dev)
+{
+        struct armada_private *priv = dev->dev_private;
+        struct armada_crtc *dcrtc = priv->dcrtc[0];
+
+        writel(0, dcrtc->base + LCD_SPU_IRQ_ENA);
+}
+
+static struct drm_ioctl_desc armada_ioctls[] = {
+        DRM_IOCTL_DEF_DRV(ARMADA_GEM_CREATE, armada_gem_create_ioctl,
+                DRM_UNLOCKED),
+        DRM_IOCTL_DEF_DRV(ARMADA_GEM_MMAP, armada_gem_mmap_ioctl,
+                DRM_UNLOCKED),
+        DRM_IOCTL_DEF_DRV(ARMADA_GEM_PWRITE, armada_gem_pwrite_ioctl,
+                DRM_UNLOCKED),
+};
+
+static const struct file_operations armada_drm_fops = {
+        .owner                  = THIS_MODULE,
+        .llseek                 = no_llseek,
+        .read                   = drm_read,
+        .poll                   = drm_poll,
+        .unlocked_ioctl         = drm_ioctl,
+        .mmap                   = drm_gem_mmap,
+        .open                   = drm_open,
+        .release                = drm_release,
+};
+
+static struct drm_driver armada_drm_driver = {
+        .load                   = armada_drm_load,
+        .open                   = NULL,
+        .preclose               = NULL,
+        .postclose              = NULL,
+        .lastclose              = NULL,
+        .unload                 = armada_drm_unload,
+        .get_vblank_counter     = drm_vblank_count,
+        .enable_vblank          = armada_drm_enable_vblank,
+        .disable_vblank         = armada_drm_disable_vblank,
+        .irq_handler            = armada_drm_irq_handler,
+        .irq_postinstall        = armada_drm_irq_postinstall,
+        .irq_uninstall          = armada_drm_irq_uninstall,
+#ifdef CONFIG_DEBUG_FS
+        .debugfs_init           = armada_drm_debugfs_init,
+        .debugfs_cleanup        = armada_drm_debugfs_cleanup,
+#endif
+        .gem_free_object        = armada_gem_free_object,
+        .prime_handle_to_fd     = drm_gem_prime_handle_to_fd,
+        .prime_fd_to_handle     = drm_gem_prime_fd_to_handle,
+        .gem_prime_export       = armada_gem_prime_export,
+        .gem_prime_import       = armada_gem_prime_import,
+        .dumb_create            = armada_gem_dumb_create,
+        .dumb_map_offset        = armada_gem_dumb_map_offset,
+        .dumb_destroy           = armada_gem_dumb_destroy,
+        .gem_vm_ops             = &armada_gem_vm_ops,
+        .major                  = 1,
+        .minor                  = 0,
+        .name                   = "armada-drm",
+        .desc                   = "Armada SoC DRM",
+        .date                   = "20120730",
+        .driver_features        = DRIVER_GEM | DRIVER_MODESET |
+                                  DRIVER_HAVE_IRQ | DRIVER_PRIME,
+        .ioctls                 = armada_ioctls,
+        .fops                   = &armada_drm_fops,
+};
+
+static int armada_drm_probe(struct platform_device *pdev)
+{
+        return drm_platform_init(&armada_drm_driver, pdev);
+}
+
+static int armada_drm_remove(struct platform_device *pdev)
+{
+        drm_platform_exit(&armada_drm_driver, pdev);
+        return 0;
+}
+
+static const struct platform_device_id armada_drm_platform_ids[] = {
+        {
+                .name           = "armada-drm",
+                .driver_data    = (unsigned long)&armada510_ops,
+        }, {
+                .name           = "armada-510-drm",
+                .driver_data    = (unsigned long)&armada510_ops,
+        },
+        { },
+};
+MODULE_DEVICE_TABLE(platform, armada_drm_platform_ids);
+
+static struct platform_driver armada_drm_platform_driver = {
+        .probe  = armada_drm_probe,
+        .remove = armada_drm_remove,
+        .driver = {
+                .name   = "armada-drm",
+                .owner  = THIS_MODULE,
+        },
+        .id_table = armada_drm_platform_ids,
+};
+
+static int __init armada_drm_init(void)
+{
+        armada_drm_driver.num_ioctls = DRM_ARRAY_SIZE(armada_ioctls);
+        return platform_driver_register(&armada_drm_platform_driver);
+}
+module_init(armada_drm_init);
+
+static void __exit armada_drm_exit(void)
+{
+        platform_driver_unregister(&armada_drm_platform_driver);
+}
+module_exit(armada_drm_exit);
+
+MODULE_AUTHOR("Russell King <rmk+kernel@arm.linux.org.uk>");
+MODULE_DESCRIPTION("Armada DRM Driver");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("platform:armada-drm");
diff --git a/drivers/gpu/drm/armada/armada_fb.c b/drivers/gpu/drm/armada/armada_fb.c
new file mode 100644
index 000000000000..1c90969def3e
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_fb.c
@@ -0,0 +1,170 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
+#include "armada_drm.h"
+#include "armada_fb.h"
+#include "armada_gem.h"
+#include "armada_hw.h"
+
+static void armada_fb_destroy(struct drm_framebuffer *fb)
+{
+        struct armada_framebuffer *dfb = drm_fb_to_armada_fb(fb);
+
+        drm_framebuffer_cleanup(&dfb->fb);
+        drm_gem_object_unreference_unlocked(&dfb->obj->obj);
+        kfree(dfb);
+}
+
+static int armada_fb_create_handle(struct drm_framebuffer *fb,
+        struct drm_file *dfile, unsigned int *handle)
+{
+        struct armada_framebuffer *dfb = drm_fb_to_armada_fb(fb);
+        return drm_gem_handle_create(dfile, &dfb->obj->obj, handle);
+}
+
+static const struct drm_framebuffer_funcs armada_fb_funcs = {
+        .destroy        = armada_fb_destroy,
+        .create_handle  = armada_fb_create_handle,
+};
+
+struct armada_framebuffer *armada_framebuffer_create(struct drm_device *dev,
+        struct drm_mode_fb_cmd2 *mode, struct armada_gem_object *obj)
+{
+        struct armada_framebuffer *dfb;
+        uint8_t format, config;
+        int ret;
+
+        switch (mode->pixel_format) {
+#define FMT(drm, fmt, mod)              \
+        case DRM_FORMAT_##drm:          \
+                format = CFG_##fmt;     \
+                config = mod;           \
+                break
+        FMT(RGB565,     565,            CFG_SWAPRB);
+        FMT(BGR565,     565,            0);
+        FMT(ARGB1555,   1555,           CFG_SWAPRB);
+        FMT(ABGR1555,   1555,           0);
+        FMT(RGB888,     888PACK,        CFG_SWAPRB);
+        FMT(BGR888,     888PACK,        0);
+        FMT(XRGB8888,   X888,           CFG_SWAPRB);
+        FMT(XBGR8888,   X888,           0);
+        FMT(ARGB8888,   8888,           CFG_SWAPRB);
+        FMT(ABGR8888,   8888,           0);
+        FMT(YUYV,       422PACK,        CFG_YUV2RGB | CFG_SWAPYU | CFG_SWAPUV);
+        FMT(UYVY,       422PACK,        CFG_YUV2RGB);
+        FMT(VYUY,       422PACK,        CFG_YUV2RGB | CFG_SWAPUV);
+        FMT(YVYU,       422PACK,        CFG_YUV2RGB | CFG_SWAPYU);
+        FMT(YUV422,     422,            CFG_YUV2RGB);
+        FMT(YVU422,     422,            CFG_YUV2RGB | CFG_SWAPUV);
+        FMT(YUV420,     420,            CFG_YUV2RGB);
+        FMT(YVU420,     420,            CFG_YUV2RGB | CFG_SWAPUV);
+        FMT(C8,         PSEUDO8,        0);
+#undef FMT
+        default:
+                return ERR_PTR(-EINVAL);
+        }
+
+        dfb = kzalloc(sizeof(*dfb), GFP_KERNEL);
+        if (!dfb) {
+                DRM_ERROR("failed to allocate Armada fb object\n");
+                return ERR_PTR(-ENOMEM);
+        }
+
+        dfb->fmt = format;
+        dfb->mod = config;
+        dfb->obj = obj;
+
+        drm_helper_mode_fill_fb_struct(&dfb->fb, mode);
+
+        ret = drm_framebuffer_init(dev, &dfb->fb, &armada_fb_funcs);
+        if (ret) {
+                kfree(dfb);
+                return ERR_PTR(ret);
+        }
+
+        /*
+         * Take a reference on our object as we're successful - the
+         * caller already holds a reference, which keeps us safe for
+         * the above call, but the caller will drop their reference
+         * to it.  Hence we need to take our own reference.
+         */
+        drm_gem_object_reference(&obj->obj);
+
+        return dfb;
+}
+
+static struct drm_framebuffer *armada_fb_create(struct drm_device *dev,
+        struct drm_file *dfile, struct drm_mode_fb_cmd2 *mode)
+{
+        struct armada_gem_object *obj;
+        struct armada_framebuffer *dfb;
+        int ret;
+
+        DRM_DEBUG_DRIVER("w%u h%u pf%08x f%u p%u,%u,%u\n",
+                mode->width, mode->height, mode->pixel_format,
+                mode->flags, mode->pitches[0], mode->pitches[1],
+                mode->pitches[2]);
+
+        /* We can only handle a single plane at the moment */
+        if (drm_format_num_planes(mode->pixel_format) > 1 &&
+            (mode->handles[0] != mode->handles[1] ||
+             mode->handles[0] != mode->handles[2])) {
+                ret = -EINVAL;
+                goto err;
+        }
+
+        obj = armada_gem_object_lookup(dev, dfile, mode->handles[0]);
+        if (!obj) {
+                ret = -ENOENT;
+                goto err;
+        }
+
+        if (obj->obj.import_attach && !obj->sgt) {
+                ret = armada_gem_map_import(obj);
+                if (ret)
+                        goto err_unref;
+        }
+
+        /* Framebuffer objects must have a valid device address for scanout */
+        if (obj->dev_addr == DMA_ERROR_CODE) {
+                ret = -EINVAL;
+                goto err_unref;
+        }
+
+        dfb = armada_framebuffer_create(dev, mode, obj);
+        if (IS_ERR(dfb)) {
+                ret = PTR_ERR(dfb);
+                goto err;
+        }
+
+        drm_gem_object_unreference_unlocked(&obj->obj);
+
+        return &dfb->fb;
+
+ err_unref:
+        drm_gem_object_unreference_unlocked(&obj->obj);
+ err:
+        DRM_ERROR("failed to initialize framebuffer: %d\n", ret);
+        return ERR_PTR(ret);
+}
+
+static void armada_output_poll_changed(struct drm_device *dev)
+{
+        struct armada_private *priv = dev->dev_private;
+        struct drm_fb_helper *fbh = priv->fbdev;
+
+        if (fbh)
+                drm_fb_helper_hotplug_event(fbh);
+}
+
+const struct drm_mode_config_funcs armada_drm_mode_config_funcs = {
+        .fb_create              = armada_fb_create,
+        .output_poll_changed    = armada_output_poll_changed,
+};
diff --git a/drivers/gpu/drm/armada/armada_fb.h b/drivers/gpu/drm/armada/armada_fb.h
new file mode 100644
index 000000000000..ce3f12ebfc53
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_fb.h
@@ -0,0 +1,24 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef ARMADA_FB_H
+#define ARMADA_FB_H
+
+struct armada_framebuffer {
+        struct drm_framebuffer  fb;
+        struct armada_gem_object *obj;
+        uint8_t                 fmt;
+        uint8_t                 mod;
+};
+#define drm_fb_to_armada_fb(dfb) \
+        container_of(dfb, struct armada_framebuffer, fb)
+#define drm_fb_obj(fb) drm_fb_to_armada_fb(fb)->obj
+
+struct armada_framebuffer *armada_framebuffer_create(struct drm_device *,
+        struct drm_mode_fb_cmd2 *, struct armada_gem_object *);
+
+#endif
diff --git a/drivers/gpu/drm/armada/armada_fbdev.c b/drivers/gpu/drm/armada/armada_fbdev.c
new file mode 100644
index 000000000000..dd5ea77dac96
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_fbdev.c
@@ -0,0 +1,202 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *  Written from the i915 driver.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/errno.h>
+#include <linux/fb.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_fb_helper.h>
+#include "armada_crtc.h"
+#include "armada_drm.h"
+#include "armada_fb.h"
+#include "armada_gem.h"
+
+static /*const*/ struct fb_ops armada_fb_ops = {
+        .owner          = THIS_MODULE,
+        .fb_check_var   = drm_fb_helper_check_var,
+        .fb_set_par     = drm_fb_helper_set_par,
+        .fb_fillrect    = cfb_fillrect,
+        .fb_copyarea    = cfb_copyarea,
+        .fb_imageblit   = cfb_imageblit,
+        .fb_pan_display = drm_fb_helper_pan_display,
+        .fb_blank       = drm_fb_helper_blank,
+        .fb_setcmap     = drm_fb_helper_setcmap,
+        .fb_debug_enter = drm_fb_helper_debug_enter,
+        .fb_debug_leave = drm_fb_helper_debug_leave,
+};
+
+static int armada_fb_create(struct drm_fb_helper *fbh,
+        struct drm_fb_helper_surface_size *sizes)
+{
+        struct drm_device *dev = fbh->dev;
+        struct drm_mode_fb_cmd2 mode;
+        struct armada_framebuffer *dfb;
+        struct armada_gem_object *obj;
+        struct fb_info *info;
+        int size, ret;
+        void *ptr;
+
+        memset(&mode, 0, sizeof(mode));
+        mode.width = sizes->surface_width;
+        mode.height = sizes->surface_height;
+        mode.pitches[0] = armada_pitch(mode.width, sizes->surface_bpp);
+        mode.pixel_format = drm_mode_legacy_fb_format(sizes->surface_bpp,
+                                        sizes->surface_depth);
+
+        size = mode.pitches[0] * mode.height;
+        obj = armada_gem_alloc_private_object(dev, size);
+        if (!obj) {
+                DRM_ERROR("failed to allocate fb memory\n");
+                return -ENOMEM;
+        }
+
+        ret = armada_gem_linear_back(dev, obj);
+        if (ret) {
+                drm_gem_object_unreference_unlocked(&obj->obj);
+                return ret;
+        }
+
+        ptr = armada_gem_map_object(dev, obj);
+        if (!ptr) {
+                drm_gem_object_unreference_unlocked(&obj->obj);
+                return -ENOMEM;
+        }
+
+        dfb = armada_framebuffer_create(dev, &mode, obj);
+
+        /*
+         * A reference is now held by the framebuffer object if
+         * successful, otherwise this drops the ref for the error path.
+         */
+        drm_gem_object_unreference_unlocked(&obj->obj);
+
+        if (IS_ERR(dfb))
+                return PTR_ERR(dfb);
+
+        info = framebuffer_alloc(0, dev->dev);
+        if (!info) {
+                ret = -ENOMEM;
+                goto err_fballoc;
+        }
+
+        ret = fb_alloc_cmap(&info->cmap, 256, 0);
+        if (ret) {
+                ret = -ENOMEM;
+                goto err_fbcmap;
+        }
+
+        strlcpy(info->fix.id, "armada-drmfb", sizeof(info->fix.id));
+        info->par = fbh;
+        info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT;
+        info->fbops = &armada_fb_ops;
+        info->fix.smem_start = obj->phys_addr;
+        info->fix.smem_len = obj->obj.size;
+        info->screen_size = obj->obj.size;
+        info->screen_base = ptr;
+        fbh->fb = &dfb->fb;
+        fbh->fbdev = info;
+        drm_fb_helper_fill_fix(info, dfb->fb.pitches[0], dfb->fb.depth);
+        drm_fb_helper_fill_var(info, fbh, sizes->fb_width, sizes->fb_height);
+
+        DRM_DEBUG_KMS("allocated %dx%d %dbpp fb: 0x%08x\n",
+                dfb->fb.width, dfb->fb.height,
+                dfb->fb.bits_per_pixel, obj->phys_addr);
+
+        return 0;
+
+ err_fbcmap:
+        framebuffer_release(info);
+ err_fballoc:
+        dfb->fb.funcs->destroy(&dfb->fb);
+        return ret;
+}
+
+static int armada_fb_probe(struct drm_fb_helper *fbh,
+        struct drm_fb_helper_surface_size *sizes)
+{
+        int ret = 0;
+
+        if (!fbh->fb) {
+                ret = armada_fb_create(fbh, sizes);
+                if (ret == 0)
+                        ret = 1;
+        }
+        return ret;
+}
+
+static struct drm_fb_helper_funcs armada_fb_helper_funcs = {
+        .gamma_set      = armada_drm_crtc_gamma_set,
+        .gamma_get      = armada_drm_crtc_gamma_get,
+        .fb_probe       = armada_fb_probe,
+};
+
+int armada_fbdev_init(struct drm_device *dev)
+{
+        struct armada_private *priv = dev->dev_private;
+        struct drm_fb_helper *fbh;
+        int ret;
+
+        fbh = devm_kzalloc(dev->dev, sizeof(*fbh), GFP_KERNEL);
+        if (!fbh)
+                return -ENOMEM;
+
+        priv->fbdev = fbh;
+
+        fbh->funcs = &armada_fb_helper_funcs;
+
+        ret = drm_fb_helper_init(dev, fbh, 1, 1);
+        if (ret) {
+                DRM_ERROR("failed to initialize drm fb helper\n");
+                goto err_fb_helper;
+        }
+
+        ret = drm_fb_helper_single_add_all_connectors(fbh);
+        if (ret) {
+                DRM_ERROR("failed to add fb connectors\n");
+                goto err_fb_setup;
+        }
+
+        ret = drm_fb_helper_initial_config(fbh, 32);
+        if (ret) {
+                DRM_ERROR("failed to set initial config\n");
+                goto err_fb_setup;
+        }
+
+        return 0;
+ err_fb_setup:
+        drm_fb_helper_fini(fbh);
+ err_fb_helper:
+        priv->fbdev = NULL;
+        return ret;
+}
+
+void armada_fbdev_fini(struct drm_device *dev)
+{
+        struct armada_private *priv = dev->dev_private;
+        struct drm_fb_helper *fbh = priv->fbdev;
+
+        if (fbh) {
+                struct fb_info *info = fbh->fbdev;
+
+                if (info) {
+                        unregister_framebuffer(info);
+                        if (info->cmap.len)
+                                fb_dealloc_cmap(&info->cmap);
+                        framebuffer_release(info);
+                }
+
+                if (fbh->fb)
+                        fbh->fb->funcs->destroy(fbh->fb);
+
+                drm_fb_helper_fini(fbh);
+
+                priv->fbdev = NULL;
+        }
+}
diff --git a/drivers/gpu/drm/armada/armada_gem.c b/drivers/gpu/drm/armada/armada_gem.c
new file mode 100644
index 000000000000..9f2356bae7fd
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_gem.c
@@ -0,0 +1,611 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <linux/dma-buf.h>
+#include <linux/dma-mapping.h>
+#include <linux/shmem_fs.h>
+#include <drm/drmP.h>
+#include "armada_drm.h"
+#include "armada_gem.h"
+#include <drm/armada_drm.h>
+#include "armada_ioctlP.h"
+
+static int armada_gem_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
+{
+        struct armada_gem_object *obj = drm_to_armada_gem(vma->vm_private_data);
+        unsigned long addr = (unsigned long)vmf->virtual_address;
+        unsigned long pfn = obj->phys_addr >> PAGE_SHIFT;
+        int ret;
+
+        pfn += (addr - vma->vm_start) >> PAGE_SHIFT;
+        ret = vm_insert_pfn(vma, addr, pfn);
+
+        switch (ret) {
+        case 0:
+        case -EBUSY:
+                return VM_FAULT_NOPAGE;
+        case -ENOMEM:
+                return VM_FAULT_OOM;
+        default:
+                return VM_FAULT_SIGBUS;
+        }
+}
+
+const struct vm_operations_struct armada_gem_vm_ops = {
+        .fault  = armada_gem_vm_fault,
+        .open   = drm_gem_vm_open,
+        .close  = drm_gem_vm_close,
+};
+
+static size_t roundup_gem_size(size_t size)
+{
+        return roundup(size, PAGE_SIZE);
+}
+
+/* dev->struct_mutex is held here */
+void armada_gem_free_object(struct drm_gem_object *obj)
+{
+        struct armada_gem_object *dobj = drm_to_armada_gem(obj);
+
+        DRM_DEBUG_DRIVER("release obj %p\n", dobj);
+
+        drm_gem_free_mmap_offset(&dobj->obj);
+
+        if (dobj->page) {
+                /* page backed memory */
+                unsigned int order = get_order(dobj->obj.size);
+                __free_pages(dobj->page, order);
+        } else if (dobj->linear) {
+                /* linear backed memory */
+                drm_mm_remove_node(dobj->linear);
+                kfree(dobj->linear);
+                if (dobj->addr)
+                        iounmap(dobj->addr);
+        }
+
+        if (dobj->obj.import_attach) {
+                /* We only ever display imported data */
+                dma_buf_unmap_attachment(dobj->obj.import_attach, dobj->sgt,
+                                         DMA_TO_DEVICE);
+                drm_prime_gem_destroy(&dobj->obj, NULL);
+        }
+
+        drm_gem_object_release(&dobj->obj);
+
+        kfree(dobj);
+}
+
+int
+armada_gem_linear_back(struct drm_device *dev, struct armada_gem_object *obj)
+{
+        struct armada_private *priv = dev->dev_private;
+        size_t size = obj->obj.size;
+
+        if (obj->page || obj->linear)
+                return 0;
+
+        /*
+         * If it is a small allocation (typically cursor, which will
+         * be 32x64 or 64x32 ARGB pixels) try to get it from the system.
+         * Framebuffers will never be this small (our minimum size for
+         * framebuffers is larger than this anyway.)  Such objects are
+         * only accessed by the CPU so we don't need any special handing
+         * here.
+         */
+        if (size <= 8192) {
+                unsigned int order = get_order(size);
+                struct page *p = alloc_pages(GFP_KERNEL, order);
+
+                if (p) {
+                        obj->addr = page_address(p);
+                        obj->phys_addr = page_to_phys(p);
+                        obj->page = p;
+
+                        memset(obj->addr, 0, PAGE_ALIGN(size));
+                }
+        }
+
+        /*
+         * We could grab something from CMA if it's enabled, but that
+         * involves building in a problem:
+         *
+         * CMA's interface uses dma_alloc_coherent(), which provides us
+         * with an CPU virtual address and a device address.
+         *
+         * The CPU virtual address may be either an address in the kernel
+         * direct mapped region (for example, as it would be on x86) or
+         * it may be remapped into another part of kernel memory space
+         * (eg, as it would be on ARM.)  This means virt_to_phys() on the
+         * returned virtual address is invalid depending on the architecture
+         * implementation.
+         *
+         * The device address may also not be a physical address; it may
+         * be that there is some kind of remapping between the device and
+         * system RAM, which makes the use of the device address also
+         * unsafe to re-use as a physical address.
+         *
+         * This makes DRM usage of dma_alloc_coherent() in a generic way
+         * at best very questionable and unsafe.
+         */
+
+        /* Otherwise, grab it from our linear allocation */
+        if (!obj->page) {
+                struct drm_mm_node *node;
+                unsigned align = min_t(unsigned, size, SZ_2M);
+                void __iomem *ptr;
+                int ret;
+
+                node = kzalloc(sizeof(*node), GFP_KERNEL);
+                if (!node)
+                        return -ENOSPC;
+
+                mutex_lock(&dev->struct_mutex);
+                ret = drm_mm_insert_node(&priv->linear, node, size, align,
+                                         DRM_MM_SEARCH_DEFAULT);
+                mutex_unlock(&dev->struct_mutex);
+                if (ret) {
+                        kfree(node);
+                        return ret;
+                }
+
+                obj->linear = node;
+
+                /* Ensure that the memory we're returning is cleared. */
+                ptr = ioremap_wc(obj->linear->start, size);
+                if (!ptr) {
+                        mutex_lock(&dev->struct_mutex);
+                        drm_mm_remove_node(obj->linear);
+                        mutex_unlock(&dev->struct_mutex);
+                        kfree(obj->linear);
+                        obj->linear = NULL;
+                        return -ENOMEM;
+                }
+
+                memset_io(ptr, 0, size);
+                iounmap(ptr);
+
+                obj->phys_addr = obj->linear->start;
+                obj->dev_addr = obj->linear->start;
+        }
+
+        DRM_DEBUG_DRIVER("obj %p phys %#x dev %#x\n",
+                         obj, obj->phys_addr, obj->dev_addr);
+
+        return 0;
+}
+
+void *
+armada_gem_map_object(struct drm_device *dev, struct armada_gem_object *dobj)
+{
+        /* only linear objects need to be ioremap'd */
+        if (!dobj->addr && dobj->linear)
+                dobj->addr = ioremap_wc(dobj->phys_addr, dobj->obj.size);
+        return dobj->addr;
+}
+
+struct armada_gem_object *
+armada_gem_alloc_private_object(struct drm_device *dev, size_t size)
+{
+        struct armada_gem_object *obj;
+
+        size = roundup_gem_size(size);
+
+        obj = kzalloc(sizeof(*obj), GFP_KERNEL);
+        if (!obj)
+                return NULL;
+
+        drm_gem_private_object_init(dev, &obj->obj, size);
+        obj->dev_addr = DMA_ERROR_CODE;
+
+        DRM_DEBUG_DRIVER("alloc private obj %p size %zu\n", obj, size);
+
+        return obj;
+}
+
+struct armada_gem_object *armada_gem_alloc_object(struct drm_device *dev,
+        size_t size)
+{
+        struct armada_gem_object *obj;
+        struct address_space *mapping;
+
+        size = roundup_gem_size(size);
+
+        obj = kzalloc(sizeof(*obj), GFP_KERNEL);
+        if (!obj)
+                return NULL;
+
+        if (drm_gem_object_init(dev, &obj->obj, size)) {
+                kfree(obj);
+                return NULL;
+        }
+
+        obj->dev_addr = DMA_ERROR_CODE;
+
+        mapping = obj->obj.filp->f_path.dentry->d_inode->i_mapping;
+        mapping_set_gfp_mask(mapping, GFP_HIGHUSER | __GFP_RECLAIMABLE);
+
+        DRM_DEBUG_DRIVER("alloc obj %p size %zu\n", obj, size);
+
+        return obj;
+}
+
+/* Dumb alloc support */
+int armada_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
+        struct drm_mode_create_dumb *args)
+{
+        struct armada_gem_object *dobj;
+        u32 handle;
+        size_t size;
+        int ret;
+
+        args->pitch = armada_pitch(args->width, args->bpp);
+        args->size = size = args->pitch * args->height;
+
+        dobj = armada_gem_alloc_private_object(dev, size);
+        if (dobj == NULL)
+                return -ENOMEM;
+
+        ret = armada_gem_linear_back(dev, dobj);
+        if (ret)
+                goto err;
+
+        ret = drm_gem_handle_create(file, &dobj->obj, &handle);
+        if (ret)
+                goto err;
+
+        args->handle = handle;
+
+        /* drop reference from allocate - handle holds it now */
+        DRM_DEBUG_DRIVER("obj %p size %zu handle %#x\n", dobj, size, handle);
+ err:
+        drm_gem_object_unreference_unlocked(&dobj->obj);
+        return ret;
+}
+
+int armada_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
+        uint32_t handle, uint64_t *offset)
+{
+        struct armada_gem_object *obj;
+        int ret = 0;
+
+        mutex_lock(&dev->struct_mutex);
+        obj = armada_gem_object_lookup(dev, file, handle);
+        if (!obj) {
+                DRM_ERROR("failed to lookup gem object\n");
+                ret = -EINVAL;
+                goto err_unlock;
+        }
+
+        /* Don't allow imported objects to be mapped */
+        if (obj->obj.import_attach) {
+                ret = -EINVAL;
+                goto err_unlock;
+        }
+
+        ret = drm_gem_create_mmap_offset(&obj->obj);
+        if (ret == 0) {
+                *offset = drm_vma_node_offset_addr(&obj->obj.vma_node);
+                DRM_DEBUG_DRIVER("handle %#x offset %llx\n", handle, *offset);
+        }
+
+        drm_gem_object_unreference(&obj->obj);
+ err_unlock:
+        mutex_unlock(&dev->struct_mutex);
+
+        return ret;
+}
+
+int armada_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev,
+        uint32_t handle)
+{
+        return drm_gem_handle_delete(file, handle);
+}
+
+/* Private driver gem ioctls */
+int armada_gem_create_ioctl(struct drm_device *dev, void *data,
+        struct drm_file *file)
+{
+        struct drm_armada_gem_create *args = data;
+        struct armada_gem_object *dobj;
+        size_t size;
+        u32 handle;
+        int ret;
+
+        if (args->size == 0)
+                return -ENOMEM;
+
+        size = args->size;
+
+        dobj = armada_gem_alloc_object(dev, size);
+        if (dobj == NULL)
+                return -ENOMEM;
+
+        ret = drm_gem_handle_create(file, &dobj->obj, &handle);
+        if (ret)
+                goto err;
+
+        args->handle = handle;
+
+        /* drop reference from allocate - handle holds it now */
+        DRM_DEBUG_DRIVER("obj %p size %zu handle %#x\n", dobj, size, handle);
+ err:
+        drm_gem_object_unreference_unlocked(&dobj->obj);
+        return ret;
+}
+
+/* Map a shmem-backed object into process memory space */
+int armada_gem_mmap_ioctl(struct drm_device *dev, void *data,
+        struct drm_file *file)
+{
+        struct drm_armada_gem_mmap *args = data;
+        struct armada_gem_object *dobj;
+        unsigned long addr;
+
+        dobj = armada_gem_object_lookup(dev, file, args->handle);
+        if (dobj == NULL)
+                return -ENOENT;
+
+        if (!dobj->obj.filp) {
+                drm_gem_object_unreference(&dobj->obj);
+                return -EINVAL;
+        }
+
+        addr = vm_mmap(dobj->obj.filp, 0, args->size, PROT_READ | PROT_WRITE,
+                       MAP_SHARED, args->offset);
+        drm_gem_object_unreference(&dobj->obj);
+        if (IS_ERR_VALUE(addr))
+                return addr;
+
+        args->addr = addr;
+
+        return 0;
+}
+
+int armada_gem_pwrite_ioctl(struct drm_device *dev, void *data,
+        struct drm_file *file)
+{
+        struct drm_armada_gem_pwrite *args = data;
+        struct armada_gem_object *dobj;
+        char __user *ptr;
+        int ret;
+
+        DRM_DEBUG_DRIVER("handle %u off %u size %u ptr 0x%llx\n",
+                args->handle, args->offset, args->size, args->ptr);
+
+        if (args->size == 0)
+                return 0;
+
+        ptr = (char __user *)(uintptr_t)args->ptr;
+
+        if (!access_ok(VERIFY_READ, ptr, args->size))
+                return -EFAULT;
+
+        ret = fault_in_multipages_readable(ptr, args->size);
+        if (ret)
+                return ret;
+
+        dobj = armada_gem_object_lookup(dev, file, args->handle);
+        if (dobj == NULL)
+                return -ENOENT;
+
+        /* Must be a kernel-mapped object */
+        if (!dobj->addr)
+                return -EINVAL;
+
+        if (args->offset > dobj->obj.size ||
+            args->size > dobj->obj.size - args->offset) {
+                DRM_ERROR("invalid size: object size %u\n", dobj->obj.size);
+                ret = -EINVAL;
+                goto unref;
+        }
+
+        if (copy_from_user(dobj->addr + args->offset, ptr, args->size)) {
+                ret = -EFAULT;
+        } else if (dobj->update) {
+                dobj->update(dobj->update_data);
+                ret = 0;
+        }
+
+ unref:
+        drm_gem_object_unreference_unlocked(&dobj->obj);
+        return ret;
+}
+
+/* Prime support */
+struct sg_table *
+armada_gem_prime_map_dma_buf(struct dma_buf_attachment *attach,
+        enum dma_data_direction dir)
+{
+        struct drm_gem_object *obj = attach->dmabuf->priv;
+        struct armada_gem_object *dobj = drm_to_armada_gem(obj);
+        struct scatterlist *sg;
+        struct sg_table *sgt;
+        int i, num;
+
+        sgt = kmalloc(sizeof(*sgt), GFP_KERNEL);
+        if (!sgt)
+                return NULL;
+
+        if (dobj->obj.filp) {
+                struct address_space *mapping;
+                gfp_t gfp;
+                int count;
+
+                count = dobj->obj.size / PAGE_SIZE;
+                if (sg_alloc_table(sgt, count, GFP_KERNEL))
+                        goto free_sgt;
+
+                mapping = file_inode(dobj->obj.filp)->i_mapping;
+                gfp = mapping_gfp_mask(mapping);
+
+                for_each_sg(sgt->sgl, sg, count, i) {
+                        struct page *page;
+
+                        page = shmem_read_mapping_page_gfp(mapping, i, gfp);
+                        if (IS_ERR(page)) {
+                                num = i;
+                                goto release;
+                        }
+
+                        sg_set_page(sg, page, PAGE_SIZE, 0);
+                }
+
+                if (dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir) == 0) {
+                        num = sgt->nents;
+                        goto release;
+                }
+        } else if (dobj->page) {
+                /* Single contiguous page */
+                if (sg_alloc_table(sgt, 1, GFP_KERNEL))
+                        goto free_sgt;
+
+                sg_set_page(sgt->sgl, dobj->page, dobj->obj.size, 0);
+
+                if (dma_map_sg(attach->dev, sgt->sgl, sgt->nents, dir) == 0)
+                        goto free_table;
+        } else if (dobj->linear) {
+                /* Single contiguous physical region - no struct page */
+                if (sg_alloc_table(sgt, 1, GFP_KERNEL))
+                        goto free_sgt;
+                sg_dma_address(sgt->sgl) = dobj->dev_addr;
+                sg_dma_len(sgt->sgl) = dobj->obj.size;
+        } else {
+                goto free_sgt;
+        }
+        return sgt;
+
+ release:
+        for_each_sg(sgt->sgl, sg, num, i)
+                page_cache_release(sg_page(sg));
+ free_table:
+        sg_free_table(sgt);
+ free_sgt:
+        kfree(sgt);
+        return NULL;
+}
+
+static void armada_gem_prime_unmap_dma_buf(struct dma_buf_attachment *attach,
+        struct sg_table *sgt, enum dma_data_direction dir)
+{
+        struct drm_gem_object *obj = attach->dmabuf->priv;
+        struct armada_gem_object *dobj = drm_to_armada_gem(obj);
+        int i;
+
+        if (!dobj->linear)
+                dma_unmap_sg(attach->dev, sgt->sgl, sgt->nents, dir);
+
+        if (dobj->obj.filp) {
+                struct scatterlist *sg;
+                for_each_sg(sgt->sgl, sg, sgt->nents, i)
+                        page_cache_release(sg_page(sg));
+        }
+
+        sg_free_table(sgt);
+        kfree(sgt);
+}
+
+static void *armada_gem_dmabuf_no_kmap(struct dma_buf *buf, unsigned long n)
+{
+        return NULL;
+}
+
+static void
+armada_gem_dmabuf_no_kunmap(struct dma_buf *buf, unsigned long n, void *addr)
+{
+}
+
+static int
+armada_gem_dmabuf_mmap(struct dma_buf *buf, struct vm_area_struct *vma)
+{
+        return -EINVAL;
+}
+
+static const struct dma_buf_ops armada_gem_prime_dmabuf_ops = {
+        .map_dma_buf    = armada_gem_prime_map_dma_buf,
+        .unmap_dma_buf  = armada_gem_prime_unmap_dma_buf,
+        .release        = drm_gem_dmabuf_release,
+        .kmap_atomic    = armada_gem_dmabuf_no_kmap,
+        .kunmap_atomic  = armada_gem_dmabuf_no_kunmap,
+        .kmap           = armada_gem_dmabuf_no_kmap,
+        .kunmap         = armada_gem_dmabuf_no_kunmap,
+        .mmap           = armada_gem_dmabuf_mmap,
+};
+
+struct dma_buf *
+armada_gem_prime_export(struct drm_device *dev, struct drm_gem_object *obj,
+        int flags)
+{
+        return dma_buf_export(obj, &armada_gem_prime_dmabuf_ops, obj->size,
+                              O_RDWR);
+}
+
+struct drm_gem_object *
+armada_gem_prime_import(struct drm_device *dev, struct dma_buf *buf)
+{
+        struct dma_buf_attachment *attach;
+        struct armada_gem_object *dobj;
+
+        if (buf->ops == &armada_gem_prime_dmabuf_ops) {
+                struct drm_gem_object *obj = buf->priv;
+                if (obj->dev == dev) {
+                        /*
+                         * Importing our own dmabuf(s) increases the
+                         * refcount on the gem object itself.
+                         */
+                        drm_gem_object_reference(obj);
+                        dma_buf_put(buf);
+                        return obj;
+                }
+        }
+
+        attach = dma_buf_attach(buf, dev->dev);
+        if (IS_ERR(attach))
+                return ERR_CAST(attach);
+
+        dobj = armada_gem_alloc_private_object(dev, buf->size);
+        if (!dobj) {
+                dma_buf_detach(buf, attach);
+                return ERR_PTR(-ENOMEM);
+        }
+
+        dobj->obj.import_attach = attach;
+
+        /*
+         * Don't call dma_buf_map_attachment() here - it maps the
+         * scatterlist immediately for DMA, and this is not always
+         * an appropriate thing to do.
+         */
+        return &dobj->obj;
+}
+
+int armada_gem_map_import(struct armada_gem_object *dobj)
+{
+        int ret;
+
+        dobj->sgt = dma_buf_map_attachment(dobj->obj.import_attach,
+                                          DMA_TO_DEVICE);
+        if (!dobj->sgt) {
+                DRM_ERROR("dma_buf_map_attachment() returned NULL\n");
+                return -EINVAL;
+        }
+        if (IS_ERR(dobj->sgt)) {
+                ret = PTR_ERR(dobj->sgt);
+                dobj->sgt = NULL;
+                DRM_ERROR("dma_buf_map_attachment() error: %d\n", ret);
+                return ret;
+        }
+        if (dobj->sgt->nents > 1) {
+                DRM_ERROR("dma_buf_map_attachment() returned an (unsupported) scattered list\n");
+                return -EINVAL;
+        }
+        if (sg_dma_len(dobj->sgt->sgl) < dobj->obj.size) {
+                DRM_ERROR("dma_buf_map_attachment() returned a small buffer\n");
+                return -EINVAL;
+        }
+        dobj->dev_addr = sg_dma_address(dobj->sgt->sgl);
+        return 0;
+}
diff --git a/drivers/gpu/drm/armada/armada_gem.h b/drivers/gpu/drm/armada/armada_gem.h
new file mode 100644
index 000000000000..00b6cd461a03
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_gem.h
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef ARMADA_GEM_H
+#define ARMADA_GEM_H
+
+/* GEM */
+struct armada_gem_object {
+        struct drm_gem_object   obj;
+        void                    *addr;
+        phys_addr_t             phys_addr;
+        resource_size_t         dev_addr;
+        struct drm_mm_node      *linear;        /* for linear backed */
+        struct page             *page;          /* for page backed */
+        struct sg_table         *sgt;           /* for imported */
+        void                    (*update)(void *);
+        void                    *update_data;
+};
+
+extern const struct vm_operations_struct armada_gem_vm_ops;
+
+#define drm_to_armada_gem(o) container_of(o, struct armada_gem_object, obj)
+
+void armada_gem_free_object(struct drm_gem_object *);
+int armada_gem_linear_back(struct drm_device *, struct armada_gem_object *);
+void *armada_gem_map_object(struct drm_device *, struct armada_gem_object *);
+struct armada_gem_object *armada_gem_alloc_private_object(struct drm_device *,
+        size_t);
+int armada_gem_dumb_create(struct drm_file *, struct drm_device *,
+        struct drm_mode_create_dumb *);
+int armada_gem_dumb_map_offset(struct drm_file *, struct drm_device *,
+        uint32_t, uint64_t *);
+int armada_gem_dumb_destroy(struct drm_file *, struct drm_device *,
+        uint32_t);
+struct dma_buf *armada_gem_prime_export(struct drm_device *dev,
+        struct drm_gem_object *obj, int flags);
+struct drm_gem_object *armada_gem_prime_import(struct drm_device *,
+        struct dma_buf *);
+int armada_gem_map_import(struct armada_gem_object *);
+
+static inline struct armada_gem_object *armada_gem_object_lookup(
+        struct drm_device *dev, struct drm_file *dfile, unsigned handle)
+{
+        struct drm_gem_object *obj = drm_gem_object_lookup(dev, dfile, handle);
+
+        return obj ? drm_to_armada_gem(obj) : NULL;
+}
+#endif
diff --git a/drivers/gpu/drm/armada/armada_hw.h b/drivers/gpu/drm/armada/armada_hw.h
new file mode 100644
index 000000000000..27319a8335e2
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_hw.h
@@ -0,0 +1,318 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *  Rewritten from the dovefb driver, and Armada510 manuals.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef ARMADA_HW_H
+#define ARMADA_HW_H
+
+/*
+ * Note: the following registers are written from IRQ context:
+ *  LCD_SPU_V_PORCH, LCD_SPU_ADV_REG, LCD_SPUT_V_H_TOTAL
+ *  LCD_SPU_DMA_START_ADDR_[YUV][01], LCD_SPU_DMA_PITCH_YC,
+ *  LCD_SPU_DMA_PITCH_UV, LCD_SPU_DMA_OVSA_HPXL_VLN,
+ *  LCD_SPU_DMA_HPXL_VLN, LCD_SPU_DZM_HPXL_VLN, LCD_SPU_DMA_CTRL0
+ */
+enum {
+        LCD_SPU_ADV_REG                 = 0x0084,       /* Armada 510 */
+        LCD_SPU_DMA_START_ADDR_Y0       = 0x00c0,
+        LCD_SPU_DMA_START_ADDR_U0       = 0x00c4,
+        LCD_SPU_DMA_START_ADDR_V0       = 0x00c8,
+        LCD_CFG_DMA_START_ADDR_0        = 0x00cc,
+        LCD_SPU_DMA_START_ADDR_Y1       = 0x00d0,
+        LCD_SPU_DMA_START_ADDR_U1       = 0x00d4,
+        LCD_SPU_DMA_START_ADDR_V1       = 0x00d8,
+        LCD_CFG_DMA_START_ADDR_1        = 0x00dc,
+        LCD_SPU_DMA_PITCH_YC            = 0x00e0,
+        LCD_SPU_DMA_PITCH_UV            = 0x00e4,
+        LCD_SPU_DMA_OVSA_HPXL_VLN       = 0x00e8,
+        LCD_SPU_DMA_HPXL_VLN            = 0x00ec,
+        LCD_SPU_DZM_HPXL_VLN            = 0x00f0,
+        LCD_CFG_GRA_START_ADDR0         = 0x00f4,
+        LCD_CFG_GRA_START_ADDR1         = 0x00f8,
+        LCD_CFG_GRA_PITCH               = 0x00fc,
+        LCD_SPU_GRA_OVSA_HPXL_VLN       = 0x0100,
+        LCD_SPU_GRA_HPXL_VLN            = 0x0104,
+        LCD_SPU_GZM_HPXL_VLN            = 0x0108,
+        LCD_SPU_HWC_OVSA_HPXL_VLN       = 0x010c,
+        LCD_SPU_HWC_HPXL_VLN            = 0x0110,
+        LCD_SPUT_V_H_TOTAL              = 0x0114,
+        LCD_SPU_V_H_ACTIVE              = 0x0118,
+        LCD_SPU_H_PORCH                 = 0x011c,
+        LCD_SPU_V_PORCH                 = 0x0120,
+        LCD_SPU_BLANKCOLOR              = 0x0124,
+        LCD_SPU_ALPHA_COLOR1            = 0x0128,
+        LCD_SPU_ALPHA_COLOR2            = 0x012c,
+        LCD_SPU_COLORKEY_Y              = 0x0130,
+        LCD_SPU_COLORKEY_U              = 0x0134,
+        LCD_SPU_COLORKEY_V              = 0x0138,
+        LCD_CFG_RDREG4F                 = 0x013c,       /* Armada 510 */
+        LCD_SPU_SPI_RXDATA              = 0x0140,
+        LCD_SPU_ISA_RXDATA              = 0x0144,
+        LCD_SPU_HWC_RDDAT               = 0x0158,
+        LCD_SPU_GAMMA_RDDAT             = 0x015c,
+        LCD_SPU_PALETTE_RDDAT           = 0x0160,
+        LCD_SPU_IOPAD_IN                = 0x0178,
+        LCD_CFG_RDREG5F                 = 0x017c,
+        LCD_SPU_SPI_CTRL                = 0x0180,
+        LCD_SPU_SPI_TXDATA              = 0x0184,
+        LCD_SPU_SMPN_CTRL               = 0x0188,
+        LCD_SPU_DMA_CTRL0               = 0x0190,
+        LCD_SPU_DMA_CTRL1               = 0x0194,
+        LCD_SPU_SRAM_CTRL               = 0x0198,
+        LCD_SPU_SRAM_WRDAT              = 0x019c,
+        LCD_SPU_SRAM_PARA0              = 0x01a0,       /* Armada 510 */
+        LCD_SPU_SRAM_PARA1              = 0x01a4,
+        LCD_CFG_SCLK_DIV                = 0x01a8,
+        LCD_SPU_CONTRAST                = 0x01ac,
+        LCD_SPU_SATURATION              = 0x01b0,
+        LCD_SPU_CBSH_HUE                = 0x01b4,
+        LCD_SPU_DUMB_CTRL               = 0x01b8,
+        LCD_SPU_IOPAD_CONTROL           = 0x01bc,
+        LCD_SPU_IRQ_ENA                 = 0x01c0,
+        LCD_SPU_IRQ_ISR                 = 0x01c4,
+};
+
+/* For LCD_SPU_ADV_REG */
+enum {
+        ADV_VSYNC_L_OFF = 0xfff << 20,
+        ADV_GRACOLORKEY = 1 << 19,
+        ADV_VIDCOLORKEY = 1 << 18,
+        ADV_HWC32BLEND  = 1 << 15,
+        ADV_HWC32ARGB   = 1 << 14,
+        ADV_HWC32ENABLE = 1 << 13,
+        ADV_VSYNCOFFEN  = 1 << 12,
+        ADV_VSYNC_H_OFF = 0xfff << 0,
+};
+
+enum {
+        CFG_565         = 0,
+        CFG_1555        = 1,
+        CFG_888PACK     = 2,
+        CFG_X888        = 3,
+        CFG_8888        = 4,
+        CFG_422PACK     = 5,
+        CFG_422         = 6,
+        CFG_420         = 7,
+        CFG_PSEUDO4     = 9,
+        CFG_PSEUDO8     = 10,
+        CFG_SWAPRB      = 1 << 4,
+        CFG_SWAPUV      = 1 << 3,
+        CFG_SWAPYU      = 1 << 2,
+        CFG_YUV2RGB     = 1 << 1,
+};
+
+/* For LCD_SPU_DMA_CTRL0 */
+enum {
+        CFG_NOBLENDING  = 1 << 31,
+        CFG_GAMMA_ENA   = 1 << 30,
+        CFG_CBSH_ENA    = 1 << 29,
+        CFG_PALETTE_ENA = 1 << 28,
+        CFG_ARBFAST_ENA = 1 << 27,
+        CFG_HWC_1BITMOD = 1 << 26,
+        CFG_HWC_1BITENA = 1 << 25,
+        CFG_HWC_ENA     = 1 << 24,
+        CFG_DMAFORMAT   = 0xf << 20,
+#define CFG_DMA_FMT(x)  ((x) << 20)
+        CFG_GRAFORMAT   = 0xf << 16,
+#define CFG_GRA_FMT(x)  ((x) << 16)
+#define CFG_GRA_MOD(x)  ((x) << 8)
+        CFG_GRA_FTOGGLE = 1 << 15,
+        CFG_GRA_HSMOOTH = 1 << 14,
+        CFG_GRA_TSTMODE = 1 << 13,
+        CFG_GRA_ENA     = 1 << 8,
+#define CFG_DMA_MOD(x)  ((x) << 0)
+        CFG_DMA_FTOGGLE = 1 << 7,
+        CFG_DMA_HSMOOTH = 1 << 6,
+        CFG_DMA_TSTMODE = 1 << 5,
+        CFG_DMA_ENA     = 1 << 0,
+};
+
+enum {
+        CKMODE_DISABLE  = 0,
+        CKMODE_Y        = 1,
+        CKMODE_U        = 2,
+        CKMODE_RGB      = 3,
+        CKMODE_V        = 4,
+        CKMODE_R        = 5,
+        CKMODE_G        = 6,
+        CKMODE_B        = 7,
+};
+
+/* For LCD_SPU_DMA_CTRL1 */
+enum {
+        CFG_FRAME_TRIG          = 1 << 31,
+        CFG_VSYNC_INV           = 1 << 27,
+        CFG_CKMODE_MASK         = 0x7 << 24,
+#define CFG_CKMODE(x)           ((x) << 24)
+        CFG_CARRY               = 1 << 23,
+        CFG_GATED_CLK           = 1 << 21,
+        CFG_PWRDN_ENA           = 1 << 20,
+        CFG_DSCALE_MASK         = 0x3 << 18,
+        CFG_DSCALE_NONE         = 0x0 << 18,
+        CFG_DSCALE_HALF         = 0x1 << 18,
+        CFG_DSCALE_QUAR         = 0x2 << 18,
+        CFG_ALPHAM_MASK         = 0x3 << 16,
+        CFG_ALPHAM_VIDEO        = 0x0 << 16,
+        CFG_ALPHAM_GRA          = 0x1 << 16,
+        CFG_ALPHAM_CFG          = 0x2 << 16,
+        CFG_ALPHA_MASK          = 0xff << 8,
+        CFG_PIXCMD_MASK         = 0xff,
+};
+
+/* For LCD_SPU_SRAM_CTRL */
+enum {
+        SRAM_READ       = 0 << 14,
+        SRAM_WRITE      = 2 << 14,
+        SRAM_INIT       = 3 << 14,
+        SRAM_HWC32_RAM1 = 0xc << 8,
+        SRAM_HWC32_RAM2 = 0xd << 8,
+        SRAM_HWC32_RAMR = SRAM_HWC32_RAM1,
+        SRAM_HWC32_RAMG = SRAM_HWC32_RAM2,
+        SRAM_HWC32_RAMB = 0xe << 8,
+        SRAM_HWC32_TRAN = 0xf << 8,
+        SRAM_HWC        = 0xf << 8,
+};
+
+/* For LCD_SPU_SRAM_PARA1 */
+enum {
+        CFG_CSB_256x32  = 1 << 15,      /* cursor */
+        CFG_CSB_256x24  = 1 << 14,      /* palette */
+        CFG_CSB_256x8   = 1 << 13,      /* gamma */
+        CFG_PDWN1920x32 = 1 << 8,       /* Armada 510: power down vscale ram */
+        CFG_PDWN256x32  = 1 << 7,       /* power down cursor */
+        CFG_PDWN256x24  = 1 << 6,       /* power down palette */
+        CFG_PDWN256x8   = 1 << 5,       /* power down gamma */
+        CFG_PDWNHWC     = 1 << 4,       /* Armada 510: power down all hwc ram */
+        CFG_PDWN32x32   = 1 << 3,       /* power down slave->smart ram */
+        CFG_PDWN16x66   = 1 << 2,       /* power down UV fifo */
+        CFG_PDWN32x66   = 1 << 1,       /* power down Y fifo */
+        CFG_PDWN64x66   = 1 << 0,       /* power down graphic fifo */
+};
+
+/* For LCD_CFG_SCLK_DIV */
+enum {
+        /* Armada 510 */
+        SCLK_510_AXI            = 0x0 << 30,
+        SCLK_510_EXTCLK0        = 0x1 << 30,
+        SCLK_510_PLL            = 0x2 << 30,
+        SCLK_510_EXTCLK1        = 0x3 << 30,
+        SCLK_510_DIV_CHANGE     = 1 << 29,
+        SCLK_510_FRAC_DIV_MASK  = 0xfff << 16,
+        SCLK_510_INT_DIV_MASK   = 0xffff << 0,
+
+        /* Armada 16x */
+        SCLK_16X_AHB            = 0x0 << 28,
+        SCLK_16X_PCLK           = 0x1 << 28,
+        SCLK_16X_AXI            = 0x4 << 28,
+        SCLK_16X_PLL            = 0x8 << 28,
+        SCLK_16X_FRAC_DIV_MASK  = 0xfff << 16,
+        SCLK_16X_INT_DIV_MASK   = 0xffff << 0,
+};
+
+/* For LCD_SPU_DUMB_CTRL */
+enum {
+        DUMB16_RGB565_0 = 0x0 << 28,
+        DUMB16_RGB565_1 = 0x1 << 28,
+        DUMB18_RGB666_0 = 0x2 << 28,
+        DUMB18_RGB666_1 = 0x3 << 28,
+        DUMB12_RGB444_0 = 0x4 << 28,
+        DUMB12_RGB444_1 = 0x5 << 28,
+        DUMB24_RGB888_0 = 0x6 << 28,
+        DUMB_BLANK      = 0x7 << 28,
+        DUMB_MASK       = 0xf << 28,
+        CFG_BIAS_OUT    = 1 << 8,
+        CFG_REV_RGB     = 1 << 7,
+        CFG_INV_CBLANK  = 1 << 6,
+        CFG_INV_CSYNC   = 1 << 5,       /* Normally active high */
+        CFG_INV_HENA    = 1 << 4,
+        CFG_INV_VSYNC   = 1 << 3,       /* Normally active high */
+        CFG_INV_HSYNC   = 1 << 2,       /* Normally active high */
+        CFG_INV_PCLK    = 1 << 1,
+        CFG_DUMB_ENA    = 1 << 0,
+};
+
+/* For LCD_SPU_IOPAD_CONTROL */
+enum {
+        CFG_VSCALE_LN_EN        = 3 << 18,
+        CFG_GRA_VM_ENA          = 1 << 15,
+        CFG_DMA_VM_ENA          = 1 << 13,
+        CFG_CMD_VM_ENA          = 1 << 11,
+        CFG_CSC_MASK            = 3 << 8,
+        CFG_CSC_YUV_CCIR709     = 1 << 9,
+        CFG_CSC_YUV_CCIR601     = 0 << 9,
+        CFG_CSC_RGB_STUDIO      = 1 << 8,
+        CFG_CSC_RGB_COMPUTER    = 0 << 8,
+        CFG_IOPAD_MASK          = 0xf << 0,
+        CFG_IOPAD_DUMB24        = 0x0 << 0,
+        CFG_IOPAD_DUMB18SPI     = 0x1 << 0,
+        CFG_IOPAD_DUMB18GPIO    = 0x2 << 0,
+        CFG_IOPAD_DUMB16SPI     = 0x3 << 0,
+        CFG_IOPAD_DUMB16GPIO    = 0x4 << 0,
+        CFG_IOPAD_DUMB12GPIO    = 0x5 << 0,
+        CFG_IOPAD_SMART18       = 0x6 << 0,
+        CFG_IOPAD_SMART16       = 0x7 << 0,
+        CFG_IOPAD_SMART8        = 0x8 << 0,
+};
+
+#define IOPAD_DUMB24                0x0
+
+/* For LCD_SPU_IRQ_ENA */
+enum {
+        DMA_FRAME_IRQ0_ENA      = 1 << 31,
+        DMA_FRAME_IRQ1_ENA      = 1 << 30,
+        DMA_FRAME_IRQ_ENA       = DMA_FRAME_IRQ0_ENA | DMA_FRAME_IRQ1_ENA,
+        DMA_FF_UNDERFLOW_ENA    = 1 << 29,
+        GRA_FRAME_IRQ0_ENA      = 1 << 27,
+        GRA_FRAME_IRQ1_ENA      = 1 << 26,
+        GRA_FRAME_IRQ_ENA       = GRA_FRAME_IRQ0_ENA | GRA_FRAME_IRQ1_ENA,
+        GRA_FF_UNDERFLOW_ENA    = 1 << 25,
+        VSYNC_IRQ_ENA           = 1 << 23,
+        DUMB_FRAMEDONE_ENA      = 1 << 22,
+        TWC_FRAMEDONE_ENA       = 1 << 21,
+        HWC_FRAMEDONE_ENA       = 1 << 20,
+        SLV_IRQ_ENA             = 1 << 19,
+        SPI_IRQ_ENA             = 1 << 18,
+        PWRDN_IRQ_ENA           = 1 << 17,
+        ERR_IRQ_ENA             = 1 << 16,
+        CLEAN_SPU_IRQ_ISR       = 0xffff,
+};
+
+/* For LCD_SPU_IRQ_ISR */
+enum {
+        DMA_FRAME_IRQ0          = 1 << 31,
+        DMA_FRAME_IRQ1          = 1 << 30,
+        DMA_FRAME_IRQ           = DMA_FRAME_IRQ0 | DMA_FRAME_IRQ1,
+        DMA_FF_UNDERFLOW        = 1 << 29,
+        GRA_FRAME_IRQ0          = 1 << 27,
+        GRA_FRAME_IRQ1          = 1 << 26,
+        GRA_FRAME_IRQ           = GRA_FRAME_IRQ0 | GRA_FRAME_IRQ1,
+        GRA_FF_UNDERFLOW        = 1 << 25,
+        VSYNC_IRQ               = 1 << 23,
+        DUMB_FRAMEDONE          = 1 << 22,
+        TWC_FRAMEDONE           = 1 << 21,
+        HWC_FRAMEDONE           = 1 << 20,
+        SLV_IRQ                 = 1 << 19,
+        SPI_IRQ                 = 1 << 18,
+        PWRDN_IRQ               = 1 << 17,
+        ERR_IRQ                 = 1 << 16,
+        DMA_FRAME_IRQ0_LEVEL    = 1 << 15,
+        DMA_FRAME_IRQ1_LEVEL    = 1 << 14,
+        DMA_FRAME_CNT_ISR       = 3 << 12,
+        GRA_FRAME_IRQ0_LEVEL    = 1 << 11,
+        GRA_FRAME_IRQ1_LEVEL    = 1 << 10,
+        GRA_FRAME_CNT_ISR       = 3 << 8,
+        VSYNC_IRQ_LEVEL         = 1 << 7,
+        DUMB_FRAMEDONE_LEVEL    = 1 << 6,
+        TWC_FRAMEDONE_LEVEL     = 1 << 5,
+        HWC_FRAMEDONE_LEVEL     = 1 << 4,
+        SLV_FF_EMPTY            = 1 << 3,
+        DMA_FF_ALLEMPTY         = 1 << 2,
+        GRA_FF_ALLEMPTY         = 1 << 1,
+        PWRDN_IRQ_LEVEL         = 1 << 0,
+};
+
+#endif
diff --git a/drivers/gpu/drm/armada/armada_ioctlP.h b/drivers/gpu/drm/armada/armada_ioctlP.h
new file mode 100644
index 000000000000..bd8c4562066c
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_ioctlP.h
@@ -0,0 +1,18 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef ARMADA_IOCTLP_H
+#define ARMADA_IOCTLP_H
+
+#define ARMADA_IOCTL_PROTO(name)\
+extern int armada_##name##_ioctl(struct drm_device *, void *, struct drm_file *)
+
+ARMADA_IOCTL_PROTO(gem_create);
+ARMADA_IOCTL_PROTO(gem_mmap);
+ARMADA_IOCTL_PROTO(gem_pwrite);
+
+#endif
diff --git a/drivers/gpu/drm/armada/armada_output.c b/drivers/gpu/drm/armada/armada_output.c
new file mode 100644
index 000000000000..d685a5421485
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_output.c
@@ -0,0 +1,158 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_encoder_slave.h>
+#include "armada_output.h"
+#include "armada_drm.h"
+
+struct armada_connector {
+        struct drm_connector conn;
+        const struct armada_output_type *type;
+};
+
+#define drm_to_armada_conn(c) container_of(c, struct armada_connector, conn)
+
+struct drm_encoder *armada_drm_connector_encoder(struct drm_connector *conn)
+{
+        struct drm_encoder *enc = conn->encoder;
+
+        return enc ? enc : drm_encoder_find(conn->dev, conn->encoder_ids[0]);
+}
+
+static enum drm_connector_status armada_drm_connector_detect(
+        struct drm_connector *conn, bool force)
+{
+        struct armada_connector *dconn = drm_to_armada_conn(conn);
+        enum drm_connector_status status = connector_status_disconnected;
+
+        if (dconn->type->detect) {
+                status = dconn->type->detect(conn, force);
+        } else {
+                struct drm_encoder *enc = armada_drm_connector_encoder(conn);
+
+                if (enc)
+                        status = encoder_helper_funcs(enc)->detect(enc, conn);
+        }
+
+        return status;
+}
+
+static void armada_drm_connector_destroy(struct drm_connector *conn)
+{
+        struct armada_connector *dconn = drm_to_armada_conn(conn);
+
+        drm_sysfs_connector_remove(conn);
+        drm_connector_cleanup(conn);
+        kfree(dconn);
+}
+
+static int armada_drm_connector_set_property(struct drm_connector *conn,
+        struct drm_property *property, uint64_t value)
+{
+        struct armada_connector *dconn = drm_to_armada_conn(conn);
+
+        if (!dconn->type->set_property)
+                return -EINVAL;
+
+        return dconn->type->set_property(conn, property, value);
+}
+
+static const struct drm_connector_funcs armada_drm_conn_funcs = {
+        .dpms           = drm_helper_connector_dpms,
+        .fill_modes     = drm_helper_probe_single_connector_modes,
+        .detect         = armada_drm_connector_detect,
+        .destroy        = armada_drm_connector_destroy,
+        .set_property   = armada_drm_connector_set_property,
+};
+
+void armada_drm_encoder_prepare(struct drm_encoder *encoder)
+{
+        encoder_helper_funcs(encoder)->dpms(encoder, DRM_MODE_DPMS_OFF);
+}
+
+void armada_drm_encoder_commit(struct drm_encoder *encoder)
+{
+        encoder_helper_funcs(encoder)->dpms(encoder, DRM_MODE_DPMS_ON);
+}
+
+bool armada_drm_encoder_mode_fixup(struct drm_encoder *encoder,
+        const struct drm_display_mode *mode, struct drm_display_mode *adjusted)
+{
+        return true;
+}
+
+/* Shouldn't this be a generic helper function? */
+int armada_drm_slave_encoder_mode_valid(struct drm_connector *conn,
+        struct drm_display_mode *mode)
+{
+        struct drm_encoder *encoder = armada_drm_connector_encoder(conn);
+        int valid = MODE_BAD;
+
+        if (encoder) {
+                struct drm_encoder_slave *slave = to_encoder_slave(encoder);
+
+                valid = slave->slave_funcs->mode_valid(encoder, mode);
+        }
+        return valid;
+}
+
+int armada_drm_slave_encoder_set_property(struct drm_connector *conn,
+        struct drm_property *property, uint64_t value)
+{
+        struct drm_encoder *encoder = armada_drm_connector_encoder(conn);
+        int rc = -EINVAL;
+
+        if (encoder) {
+                struct drm_encoder_slave *slave = to_encoder_slave(encoder);
+
+                rc = slave->slave_funcs->set_property(encoder, conn, property,
+                                                      value);
+        }
+        return rc;
+}
+
+int armada_output_create(struct drm_device *dev,
+        const struct armada_output_type *type, const void *data)
+{
+        struct armada_connector *dconn;
+        int ret;
+
+        dconn = kzalloc(sizeof(*dconn), GFP_KERNEL);
+        if (!dconn)
+                return -ENOMEM;
+
+        dconn->type = type;
+
+        ret = drm_connector_init(dev, &dconn->conn, &armada_drm_conn_funcs,
+                                 type->connector_type);
+        if (ret) {
+                DRM_ERROR("unable to init connector\n");
+                goto err_destroy_dconn;
+        }
+
+        ret = type->create(&dconn->conn, data);
+        if (ret)
+                goto err_conn;
+
+        ret = drm_sysfs_connector_add(&dconn->conn);
+        if (ret)
+                goto err_sysfs;
+
+        return 0;
+
+ err_sysfs:
+        if (dconn->conn.encoder)
+                dconn->conn.encoder->funcs->destroy(dconn->conn.encoder);
+ err_conn:
+        drm_connector_cleanup(&dconn->conn);
+ err_destroy_dconn:
+        kfree(dconn);
+        return ret;
+}
diff --git a/drivers/gpu/drm/armada/armada_output.h b/drivers/gpu/drm/armada/armada_output.h
new file mode 100644
index 000000000000..4126d43b5057
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_output.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef ARMADA_CONNETOR_H
+#define ARMADA_CONNETOR_H
+
+#define encoder_helper_funcs(encoder) \
+        ((struct drm_encoder_helper_funcs *)encoder->helper_private)
+
+struct armada_output_type {
+        int connector_type;
+        enum drm_connector_status (*detect)(struct drm_connector *, bool);
+        int (*create)(struct drm_connector *, const void *);
+        int (*set_property)(struct drm_connector *, struct drm_property *,
+                            uint64_t);
+};
+
+struct drm_encoder *armada_drm_connector_encoder(struct drm_connector *conn);
+
+void armada_drm_encoder_prepare(struct drm_encoder *encoder);
+void armada_drm_encoder_commit(struct drm_encoder *encoder);
+
+bool armada_drm_encoder_mode_fixup(struct drm_encoder *encoder,
+        const struct drm_display_mode *mode, struct drm_display_mode *adj);
+
+int armada_drm_slave_encoder_mode_valid(struct drm_connector *conn,
+        struct drm_display_mode *mode);
+
+int armada_drm_slave_encoder_set_property(struct drm_connector *conn,
+        struct drm_property *property, uint64_t value);
+
+int armada_output_create(struct drm_device *dev,
+        const struct armada_output_type *type, const void *data);
+
+#endif
diff --git a/drivers/gpu/drm/armada/armada_overlay.c b/drivers/gpu/drm/armada/armada_overlay.c
new file mode 100644
index 000000000000..c5b06fdb459c
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_overlay.c
@@ -0,0 +1,477 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *  Rewritten from the dovefb driver, and Armada510 manuals.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <drm/drmP.h>
+#include "armada_crtc.h"
+#include "armada_drm.h"
+#include "armada_fb.h"
+#include "armada_gem.h"
+#include "armada_hw.h"
+#include <drm/armada_drm.h>
+#include "armada_ioctlP.h"
+
+struct armada_plane_properties {
+        uint32_t colorkey_yr;
+        uint32_t colorkey_ug;
+        uint32_t colorkey_vb;
+#define K2R(val) (((val) >> 0) & 0xff)
+#define K2G(val) (((val) >> 8) & 0xff)
+#define K2B(val) (((val) >> 16) & 0xff)
+        int16_t  brightness;
+        uint16_t contrast;
+        uint16_t saturation;
+        uint32_t colorkey_mode;
+};
+
+struct armada_plane {
+        struct drm_plane base;
+        spinlock_t lock;
+        struct drm_framebuffer *old_fb;
+        uint32_t src_hw;
+        uint32_t dst_hw;
+        uint32_t dst_yx;
+        uint32_t ctrl0;
+        struct {
+                struct armada_vbl_event update;
+                struct armada_regs regs[13];
+                wait_queue_head_t wait;
+        } vbl;
+        struct armada_plane_properties prop;
+};
+#define drm_to_armada_plane(p) container_of(p, struct armada_plane, base)
+
+
+static void
+armada_ovl_update_attr(struct armada_plane_properties *prop,
+        struct armada_crtc *dcrtc)
+{
+        writel_relaxed(prop->colorkey_yr, dcrtc->base + LCD_SPU_COLORKEY_Y);
+        writel_relaxed(prop->colorkey_ug, dcrtc->base + LCD_SPU_COLORKEY_U);
+        writel_relaxed(prop->colorkey_vb, dcrtc->base + LCD_SPU_COLORKEY_V);
+
+        writel_relaxed(prop->brightness << 16 | prop->contrast,
+                       dcrtc->base + LCD_SPU_CONTRAST);
+        /* Docs say 15:0, but it seems to actually be 31:16 on Armada 510 */
+        writel_relaxed(prop->saturation << 16,
+                       dcrtc->base + LCD_SPU_SATURATION);
+        writel_relaxed(0x00002000, dcrtc->base + LCD_SPU_CBSH_HUE);
+
+        spin_lock_irq(&dcrtc->irq_lock);
+        armada_updatel(prop->colorkey_mode | CFG_ALPHAM_GRA,
+                     CFG_CKMODE_MASK | CFG_ALPHAM_MASK | CFG_ALPHA_MASK,
+                     dcrtc->base + LCD_SPU_DMA_CTRL1);
+
+        armada_updatel(ADV_GRACOLORKEY, 0, dcrtc->base + LCD_SPU_ADV_REG);
+        spin_unlock_irq(&dcrtc->irq_lock);
+}
+
+/* === Plane support === */
+static void armada_plane_vbl(struct armada_crtc *dcrtc, void *data)
+{
+        struct armada_plane *dplane = data;
+        struct drm_framebuffer *fb;
+
+        armada_drm_crtc_update_regs(dcrtc, dplane->vbl.regs);
+
+        spin_lock(&dplane->lock);
+        fb = dplane->old_fb;
+        dplane->old_fb = NULL;
+        spin_unlock(&dplane->lock);
+
+        if (fb)
+                armada_drm_queue_unref_work(dcrtc->crtc.dev, fb);
+}
+
+static unsigned armada_limit(int start, unsigned size, unsigned max)
+{
+        int end = start + size;
+        if (end < 0)
+                return 0;
+        if (start < 0)
+                start = 0;
+        return (unsigned)end > max ? max - start : end - start;
+}
+
+static int
+armada_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
+        struct drm_framebuffer *fb,
+        int crtc_x, int crtc_y, unsigned crtc_w, unsigned crtc_h,
+        uint32_t src_x, uint32_t src_y, uint32_t src_w, uint32_t src_h)
+{
+        struct armada_plane *dplane = drm_to_armada_plane(plane);
+        struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
+        uint32_t val, ctrl0;
+        unsigned idx = 0;
+        int ret;
+
+        crtc_w = armada_limit(crtc_x, crtc_w, dcrtc->crtc.mode.hdisplay);
+        crtc_h = armada_limit(crtc_y, crtc_h, dcrtc->crtc.mode.vdisplay);
+        ctrl0 = CFG_DMA_FMT(drm_fb_to_armada_fb(fb)->fmt) |
+                CFG_DMA_MOD(drm_fb_to_armada_fb(fb)->mod) |
+                CFG_CBSH_ENA | CFG_DMA_HSMOOTH | CFG_DMA_ENA;
+
+        /* Does the position/size result in nothing to display? */
+        if (crtc_w == 0 || crtc_h == 0) {
+                ctrl0 &= ~CFG_DMA_ENA;
+        }
+
+        /*
+         * FIXME: if the starting point is off screen, we need to
+         * adjust src_x, src_y, src_w, src_h appropriately, and
+         * according to the scale.
+         */
+
+        if (!dcrtc->plane) {
+                dcrtc->plane = plane;
+                armada_ovl_update_attr(&dplane->prop, dcrtc);
+        }
+
+        /* FIXME: overlay on an interlaced display */
+        /* Just updating the position/size? */
+        if (plane->fb == fb && dplane->ctrl0 == ctrl0) {
+                val = (src_h & 0xffff0000) | src_w >> 16;
+                dplane->src_hw = val;
+                writel_relaxed(val, dcrtc->base + LCD_SPU_DMA_HPXL_VLN);
+                val = crtc_h << 16 | crtc_w;
+                dplane->dst_hw = val;
+                writel_relaxed(val, dcrtc->base + LCD_SPU_DZM_HPXL_VLN);
+                val = crtc_y << 16 | crtc_x;
+                dplane->dst_yx = val;
+                writel_relaxed(val, dcrtc->base + LCD_SPU_DMA_OVSA_HPXL_VLN);
+                return 0;
+        } else if (~dplane->ctrl0 & ctrl0 & CFG_DMA_ENA) {
+                /* Power up the Y/U/V FIFOs on ENA 0->1 transitions */
+                armada_updatel(0, CFG_PDWN16x66 | CFG_PDWN32x66,
+                               dcrtc->base + LCD_SPU_SRAM_PARA1);
+        }
+
+        ret = wait_event_timeout(dplane->vbl.wait,
+                                 list_empty(&dplane->vbl.update.node),
+                                 HZ/25);
+        if (ret < 0)
+                return ret;
+
+        if (plane->fb != fb) {
+                struct armada_gem_object *obj = drm_fb_obj(fb);
+                uint32_t sy, su, sv;
+
+                /*
+                 * Take a reference on the new framebuffer - we want to
+                 * hold on to it while the hardware is displaying it.
+                 */
+                drm_framebuffer_reference(fb);
+
+                if (plane->fb) {
+                        struct drm_framebuffer *older_fb;
+
+                        spin_lock_irq(&dplane->lock);
+                        older_fb = dplane->old_fb;
+                        dplane->old_fb = plane->fb;
+                        spin_unlock_irq(&dplane->lock);
+                        if (older_fb)
+                                armada_drm_queue_unref_work(dcrtc->crtc.dev,
+                                                            older_fb);
+                }
+
+                src_y >>= 16;
+                src_x >>= 16;
+                sy = obj->dev_addr + fb->offsets[0] + src_y * fb->pitches[0] +
+                        src_x * fb->bits_per_pixel / 8;
+                su = obj->dev_addr + fb->offsets[1] + src_y * fb->pitches[1] +
+                        src_x;
+                sv = obj->dev_addr + fb->offsets[2] + src_y * fb->pitches[2] +
+                        src_x;
+
+                armada_reg_queue_set(dplane->vbl.regs, idx, sy,
+                                     LCD_SPU_DMA_START_ADDR_Y0);
+                armada_reg_queue_set(dplane->vbl.regs, idx, su,
+                                     LCD_SPU_DMA_START_ADDR_U0);
+                armada_reg_queue_set(dplane->vbl.regs, idx, sv,
+                                     LCD_SPU_DMA_START_ADDR_V0);
+                armada_reg_queue_set(dplane->vbl.regs, idx, sy,
+                                     LCD_SPU_DMA_START_ADDR_Y1);
+                armada_reg_queue_set(dplane->vbl.regs, idx, su,
+                                     LCD_SPU_DMA_START_ADDR_U1);
+                armada_reg_queue_set(dplane->vbl.regs, idx, sv,
+                                     LCD_SPU_DMA_START_ADDR_V1);
+
+                val = fb->pitches[0] << 16 | fb->pitches[0];
+                armada_reg_queue_set(dplane->vbl.regs, idx, val,
+                                     LCD_SPU_DMA_PITCH_YC);
+                val = fb->pitches[1] << 16 | fb->pitches[2];
+                armada_reg_queue_set(dplane->vbl.regs, idx, val,
+                                     LCD_SPU_DMA_PITCH_UV);
+        }
+
+        val = (src_h & 0xffff0000) | src_w >> 16;
+        if (dplane->src_hw != val) {
+                dplane->src_hw = val;
+                armada_reg_queue_set(dplane->vbl.regs, idx, val,
+                                     LCD_SPU_DMA_HPXL_VLN);
+        }
+        val = crtc_h << 16 | crtc_w;
+        if (dplane->dst_hw != val) {
+                dplane->dst_hw = val;
+                armada_reg_queue_set(dplane->vbl.regs, idx, val,
+                                     LCD_SPU_DZM_HPXL_VLN);
+        }
+        val = crtc_y << 16 | crtc_x;
+        if (dplane->dst_yx != val) {
+                dplane->dst_yx = val;
+                armada_reg_queue_set(dplane->vbl.regs, idx, val,
+                                     LCD_SPU_DMA_OVSA_HPXL_VLN);
+        }
+        if (dplane->ctrl0 != ctrl0) {
+                dplane->ctrl0 = ctrl0;
+                armada_reg_queue_mod(dplane->vbl.regs, idx, ctrl0,
+                        CFG_CBSH_ENA | CFG_DMAFORMAT | CFG_DMA_FTOGGLE |
+                        CFG_DMA_HSMOOTH | CFG_DMA_TSTMODE |
+                        CFG_DMA_MOD(CFG_SWAPRB | CFG_SWAPUV | CFG_SWAPYU |
+                        CFG_YUV2RGB) | CFG_DMA_ENA,
+                        LCD_SPU_DMA_CTRL0);
+        }
+        if (idx) {
+                armada_reg_queue_end(dplane->vbl.regs, idx);
+                armada_drm_vbl_event_add(dcrtc, &dplane->vbl.update);
+        }
+        return 0;
+}
+
+static int armada_plane_disable(struct drm_plane *plane)
+{
+        struct armada_plane *dplane = drm_to_armada_plane(plane);
+        struct drm_framebuffer *fb;
+        struct armada_crtc *dcrtc;
+
+        if (!dplane->base.crtc)
+                return 0;
+
+        dcrtc = drm_to_armada_crtc(dplane->base.crtc);
+        dcrtc->plane = NULL;
+
+        spin_lock_irq(&dcrtc->irq_lock);
+        armada_drm_vbl_event_remove(dcrtc, &dplane->vbl.update);
+        armada_updatel(0, CFG_DMA_ENA, dcrtc->base + LCD_SPU_DMA_CTRL0);
+        dplane->ctrl0 = 0;
+        spin_unlock_irq(&dcrtc->irq_lock);
+
+        /* Power down the Y/U/V FIFOs */
+        armada_updatel(CFG_PDWN16x66 | CFG_PDWN32x66, 0,
+                       dcrtc->base + LCD_SPU_SRAM_PARA1);
+
+        if (plane->fb)
+                drm_framebuffer_unreference(plane->fb);
+
+        spin_lock_irq(&dplane->lock);
+        fb = dplane->old_fb;
+        dplane->old_fb = NULL;
+        spin_unlock_irq(&dplane->lock);
+        if (fb)
+                drm_framebuffer_unreference(fb);
+
+        return 0;
+}
+
+static void armada_plane_destroy(struct drm_plane *plane)
+{
+        kfree(plane);
+}
+
+static int armada_plane_set_property(struct drm_plane *plane,
+        struct drm_property *property, uint64_t val)
+{
+        struct armada_private *priv = plane->dev->dev_private;
+        struct armada_plane *dplane = drm_to_armada_plane(plane);
+        bool update_attr = false;
+
+        if (property == priv->colorkey_prop) {
+#define CCC(v) ((v) << 24 | (v) << 16 | (v) << 8)
+                dplane->prop.colorkey_yr = CCC(K2R(val));
+                dplane->prop.colorkey_ug = CCC(K2G(val));
+                dplane->prop.colorkey_vb = CCC(K2B(val));
+#undef CCC
+                update_attr = true;
+        } else if (property == priv->colorkey_min_prop) {
+                dplane->prop.colorkey_yr &= ~0x00ff0000;
+                dplane->prop.colorkey_yr |= K2R(val) << 16;
+                dplane->prop.colorkey_ug &= ~0x00ff0000;
+                dplane->prop.colorkey_ug |= K2G(val) << 16;
+                dplane->prop.colorkey_vb &= ~0x00ff0000;
+                dplane->prop.colorkey_vb |= K2B(val) << 16;
+                update_attr = true;
+        } else if (property == priv->colorkey_max_prop) {
+                dplane->prop.colorkey_yr &= ~0xff000000;
+                dplane->prop.colorkey_yr |= K2R(val) << 24;
+                dplane->prop.colorkey_ug &= ~0xff000000;
+                dplane->prop.colorkey_ug |= K2G(val) << 24;
+                dplane->prop.colorkey_vb &= ~0xff000000;
+                dplane->prop.colorkey_vb |= K2B(val) << 24;
+                update_attr = true;
+        } else if (property == priv->colorkey_val_prop) {
+                dplane->prop.colorkey_yr &= ~0x0000ff00;
+                dplane->prop.colorkey_yr |= K2R(val) << 8;
+                dplane->prop.colorkey_ug &= ~0x0000ff00;
+                dplane->prop.colorkey_ug |= K2G(val) << 8;
+                dplane->prop.colorkey_vb &= ~0x0000ff00;
+                dplane->prop.colorkey_vb |= K2B(val) << 8;
+                update_attr = true;
+        } else if (property == priv->colorkey_alpha_prop) {
+                dplane->prop.colorkey_yr &= ~0x000000ff;
+                dplane->prop.colorkey_yr |= K2R(val);
+                dplane->prop.colorkey_ug &= ~0x000000ff;
+                dplane->prop.colorkey_ug |= K2G(val);
+                dplane->prop.colorkey_vb &= ~0x000000ff;
+                dplane->prop.colorkey_vb |= K2B(val);
+                update_attr = true;
+        } else if (property == priv->colorkey_mode_prop) {
+                dplane->prop.colorkey_mode &= ~CFG_CKMODE_MASK;
+                dplane->prop.colorkey_mode |= CFG_CKMODE(val);
+                update_attr = true;
+        } else if (property == priv->brightness_prop) {
+                dplane->prop.brightness = val - 256;
+                update_attr = true;
+        } else if (property == priv->contrast_prop) {
+                dplane->prop.contrast = val;
+                update_attr = true;
+        } else if (property == priv->saturation_prop) {
+                dplane->prop.saturation = val;
+                update_attr = true;
+        }
+
+        if (update_attr && dplane->base.crtc)
+                armada_ovl_update_attr(&dplane->prop,
+                                       drm_to_armada_crtc(dplane->base.crtc));
+
+        return 0;
+}
+
+static const struct drm_plane_funcs armada_plane_funcs = {
+        .update_plane   = armada_plane_update,
+        .disable_plane  = armada_plane_disable,
+        .destroy        = armada_plane_destroy,
+        .set_property   = armada_plane_set_property,
+};
+
+static const uint32_t armada_formats[] = {
+        DRM_FORMAT_UYVY,
+        DRM_FORMAT_YUYV,
+        DRM_FORMAT_YUV420,
+        DRM_FORMAT_YVU420,
+        DRM_FORMAT_YUV422,
+        DRM_FORMAT_YVU422,
+        DRM_FORMAT_VYUY,
+        DRM_FORMAT_YVYU,
+        DRM_FORMAT_ARGB8888,
+        DRM_FORMAT_ABGR8888,
+        DRM_FORMAT_XRGB8888,
+        DRM_FORMAT_XBGR8888,
+        DRM_FORMAT_RGB888,
+        DRM_FORMAT_BGR888,
+        DRM_FORMAT_ARGB1555,
+        DRM_FORMAT_ABGR1555,
+        DRM_FORMAT_RGB565,
+        DRM_FORMAT_BGR565,
+};
+
+static struct drm_prop_enum_list armada_drm_colorkey_enum_list[] = {
+        { CKMODE_DISABLE, "disabled" },
+        { CKMODE_Y,       "Y component" },
+        { CKMODE_U,       "U component" },
+        { CKMODE_V,       "V component" },
+        { CKMODE_RGB,     "RGB" },
+        { CKMODE_R,       "R component" },
+        { CKMODE_G,       "G component" },
+        { CKMODE_B,       "B component" },
+};
+
+static int armada_overlay_create_properties(struct drm_device *dev)
+{
+        struct armada_private *priv = dev->dev_private;
+
+        if (priv->colorkey_prop)
+                return 0;
+
+        priv->colorkey_prop = drm_property_create_range(dev, 0,
+                                "colorkey", 0, 0xffffff);
+        priv->colorkey_min_prop = drm_property_create_range(dev, 0,
+                                "colorkey_min", 0, 0xffffff);
+        priv->colorkey_max_prop = drm_property_create_range(dev, 0,
+                                "colorkey_max", 0, 0xffffff);
+        priv->colorkey_val_prop = drm_property_create_range(dev, 0,
+                                "colorkey_val", 0, 0xffffff);
+        priv->colorkey_alpha_prop = drm_property_create_range(dev, 0,
+                                "colorkey_alpha", 0, 0xffffff);
+        priv->colorkey_mode_prop = drm_property_create_enum(dev, 0,
+                                "colorkey_mode",
+                                armada_drm_colorkey_enum_list,
+                                ARRAY_SIZE(armada_drm_colorkey_enum_list));
+        priv->brightness_prop = drm_property_create_range(dev, 0,
+                                "brightness", 0, 256 + 255);
+        priv->contrast_prop = drm_property_create_range(dev, 0,
+                                "contrast", 0, 0x7fff);
+        priv->saturation_prop = drm_property_create_range(dev, 0,
+                                "saturation", 0, 0x7fff);
+
+        if (!priv->colorkey_prop)
+                return -ENOMEM;
+
+        return 0;
+}
+
+int armada_overlay_plane_create(struct drm_device *dev, unsigned long crtcs)
+{
+        struct armada_private *priv = dev->dev_private;
+        struct drm_mode_object *mobj;
+        struct armada_plane *dplane;
+        int ret;
+
+        ret = armada_overlay_create_properties(dev);
+        if (ret)
+                return ret;
+
+        dplane = kzalloc(sizeof(*dplane), GFP_KERNEL);
+        if (!dplane)
+                return -ENOMEM;
+
+        spin_lock_init(&dplane->lock);
+        init_waitqueue_head(&dplane->vbl.wait);
+        armada_drm_vbl_event_init(&dplane->vbl.update, armada_plane_vbl,
+                                  dplane);
+
+        drm_plane_init(dev, &dplane->base, crtcs, &armada_plane_funcs,
+                       armada_formats, ARRAY_SIZE(armada_formats), false);
+
+        dplane->prop.colorkey_yr = 0xfefefe00;
+        dplane->prop.colorkey_ug = 0x01010100;
+        dplane->prop.colorkey_vb = 0x01010100;
+        dplane->prop.colorkey_mode = CFG_CKMODE(CKMODE_RGB);
+        dplane->prop.brightness = 0;
+        dplane->prop.contrast = 0x4000;
+        dplane->prop.saturation = 0x4000;
+
+        mobj = &dplane->base.base;
+        drm_object_attach_property(mobj, priv->colorkey_prop,
+                                   0x0101fe);
+        drm_object_attach_property(mobj, priv->colorkey_min_prop,
+                                   0x0101fe);
+        drm_object_attach_property(mobj, priv->colorkey_max_prop,
+                                   0x0101fe);
+        drm_object_attach_property(mobj, priv->colorkey_val_prop,
+                                   0x0101fe);
+        drm_object_attach_property(mobj, priv->colorkey_alpha_prop,
+                                   0x000000);
+        drm_object_attach_property(mobj, priv->colorkey_mode_prop,
+                                   CKMODE_RGB);
+        drm_object_attach_property(mobj, priv->brightness_prop, 256);
+        drm_object_attach_property(mobj, priv->contrast_prop,
+                                   dplane->prop.contrast);
+        drm_object_attach_property(mobj, priv->saturation_prop,
+                                   dplane->prop.saturation);
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/armada/armada_slave.c b/drivers/gpu/drm/armada/armada_slave.c
new file mode 100644
index 000000000000..00d0facb42f3
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_slave.c
@@ -0,0 +1,139 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *  Rewritten from the dovefb driver, and Armada510 manuals.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_encoder_slave.h>
+#include "armada_drm.h"
+#include "armada_output.h"
+#include "armada_slave.h"
+
+static int armada_drm_slave_get_modes(struct drm_connector *conn)
+{
+        struct drm_encoder *enc = armada_drm_connector_encoder(conn);
+        int count = 0;
+
+        if (enc) {
+                struct drm_encoder_slave *slave = to_encoder_slave(enc);
+
+                count = slave->slave_funcs->get_modes(enc, conn);
+        }
+
+        return count;
+}
+
+static void armada_drm_slave_destroy(struct drm_encoder *enc)
+{
+        struct drm_encoder_slave *slave = to_encoder_slave(enc);
+        struct i2c_client *client = drm_i2c_encoder_get_client(enc);
+
+        if (slave->slave_funcs)
+                slave->slave_funcs->destroy(enc);
+        if (client)
+                i2c_put_adapter(client->adapter);
+
+        drm_encoder_cleanup(&slave->base);
+        kfree(slave);
+}
+
+static const struct drm_encoder_funcs armada_drm_slave_encoder_funcs = {
+        .destroy        = armada_drm_slave_destroy,
+};
+
+static const struct drm_connector_helper_funcs armada_drm_slave_helper_funcs = {
+        .get_modes      = armada_drm_slave_get_modes,
+        .mode_valid     = armada_drm_slave_encoder_mode_valid,
+        .best_encoder   = armada_drm_connector_encoder,
+};
+
+static const struct drm_encoder_helper_funcs drm_slave_encoder_helpers = {
+        .dpms = drm_i2c_encoder_dpms,
+        .save = drm_i2c_encoder_save,
+        .restore = drm_i2c_encoder_restore,
+        .mode_fixup = drm_i2c_encoder_mode_fixup,
+        .prepare = drm_i2c_encoder_prepare,
+        .commit = drm_i2c_encoder_commit,
+        .mode_set = drm_i2c_encoder_mode_set,
+        .detect = drm_i2c_encoder_detect,
+};
+
+static int
+armada_drm_conn_slave_create(struct drm_connector *conn, const void *data)
+{
+        const struct armada_drm_slave_config *config = data;
+        struct drm_encoder_slave *slave;
+        struct i2c_adapter *adap;
+        int ret;
+
+        conn->interlace_allowed = config->interlace_allowed;
+        conn->doublescan_allowed = config->doublescan_allowed;
+        conn->polled = config->polled;
+
+        drm_connector_helper_add(conn, &armada_drm_slave_helper_funcs);
+
+        slave = kzalloc(sizeof(*slave), GFP_KERNEL);
+        if (!slave)
+                return -ENOMEM;
+
+        slave->base.possible_crtcs = config->crtcs;
+
+        adap = i2c_get_adapter(config->i2c_adapter_id);
+        if (!adap) {
+                kfree(slave);
+                return -EPROBE_DEFER;
+        }
+
+        ret = drm_encoder_init(conn->dev, &slave->base,
+                               &armada_drm_slave_encoder_funcs,
+                               DRM_MODE_ENCODER_TMDS);
+        if (ret) {
+                DRM_ERROR("unable to init encoder\n");
+                i2c_put_adapter(adap);
+                kfree(slave);
+                return ret;
+        }
+
+        ret = drm_i2c_encoder_init(conn->dev, slave, adap, &config->info);
+        i2c_put_adapter(adap);
+        if (ret) {
+                DRM_ERROR("unable to init encoder slave\n");
+                armada_drm_slave_destroy(&slave->base);
+                return ret;
+        }
+
+        drm_encoder_helper_add(&slave->base, &drm_slave_encoder_helpers);
+
+        ret = slave->slave_funcs->create_resources(&slave->base, conn);
+        if (ret) {
+                armada_drm_slave_destroy(&slave->base);
+                return ret;
+        }
+
+        ret = drm_mode_connector_attach_encoder(conn, &slave->base);
+        if (ret) {
+                armada_drm_slave_destroy(&slave->base);
+                return ret;
+        }
+
+        conn->encoder = &slave->base;
+
+        return ret;
+}
+
+static const struct armada_output_type armada_drm_conn_slave = {
+        .connector_type = DRM_MODE_CONNECTOR_HDMIA,
+        .create         = armada_drm_conn_slave_create,
+        .set_property   = armada_drm_slave_encoder_set_property,
+};
+
+int armada_drm_connector_slave_create(struct drm_device *dev,
+        const struct armada_drm_slave_config *config)
+{
+        return armada_output_create(dev, &armada_drm_conn_slave, config);
+}
diff --git a/drivers/gpu/drm/armada/armada_slave.h b/drivers/gpu/drm/armada/armada_slave.h
new file mode 100644
index 000000000000..bf2374c96fc1
--- /dev/null
+++ b/drivers/gpu/drm/armada/armada_slave.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (C) 2012 Russell King
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef ARMADA_SLAVE_H
+#define ARMADA_SLAVE_H
+
+#include <linux/i2c.h>
+#include <drm/drmP.h>
+
+struct armada_drm_slave_config {
+        int i2c_adapter_id;
+        uint32_t crtcs;
+        uint8_t polled;
+        bool interlace_allowed;
+        bool doublescan_allowed;
+        struct i2c_board_info info;
+};
+
+int armada_drm_connector_slave_create(struct drm_device *dev,
+        const struct armada_drm_slave_config *);
+
+#endif
diff --git a/drivers/gpu/drm/cirrus/cirrus_mode.c b/drivers/gpu/drm/cirrus/cirrus_mode.c
index 60685b21cc36..adabc3daaa5b 100644
--- a/drivers/gpu/drm/cirrus/cirrus_mode.c
+++ b/drivers/gpu/drm/cirrus/cirrus_mode.c
@@ -494,13 +494,12 @@ static struct drm_encoder *cirrus_encoder_init(struct drm_device *dev)
 
 int cirrus_vga_get_modes(struct drm_connector *connector)
 {
-        /* Just add a static list of modes */
-        drm_add_modes_noedid(connector, 640, 480);
-        drm_add_modes_noedid(connector, 800, 600);
-        drm_add_modes_noedid(connector, 1024, 768);
-        drm_add_modes_noedid(connector, 1280, 1024);
+        int count;
 
-        return 4;
+        /* Just add a static list of modes */
+        count = drm_add_modes_noedid(connector, 1280, 1024);
+        drm_set_preferred_mode(connector, 1024, 768);
+        return count;
 }
 
 static int cirrus_vga_mode_valid(struct drm_connector *connector,
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index d7a8370e3cdc..d6cf77c472e7 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -1303,7 +1303,7 @@ static void drm_crtc_convert_to_umode(struct drm_mode_modeinfo *out,
 }
 
 /**
- * drm_crtc_convert_to_umode - convert a modeinfo into a drm_display_mode
+ * drm_crtc_convert_umode - convert a modeinfo into a drm_display_mode
  * @out: drm_display_mode to return to the user
  * @in: drm_mode_modeinfo to use
  *
@@ -1557,7 +1557,7 @@ int drm_mode_getcrtc(struct drm_device *dev,
         obj = drm_mode_object_find(dev, crtc_resp->crtc_id,
                                    DRM_MODE_OBJECT_CRTC);
         if (!obj) {
-                ret = -EINVAL;
+                ret = -ENOENT;
                 goto out;
         }
         crtc = obj_to_crtc(obj);
@@ -1641,7 +1641,7 @@ int drm_mode_getconnector(struct drm_device *dev, void *data,
         obj = drm_mode_object_find(dev, out_resp->connector_id,
                                    DRM_MODE_OBJECT_CONNECTOR);
         if (!obj) {
-                ret = -EINVAL;
+                ret = -ENOENT;
                 goto out;
         }
         connector = obj_to_connector(obj);
@@ -1757,7 +1757,7 @@ int drm_mode_getencoder(struct drm_device *dev, void *data,
         obj = drm_mode_object_find(dev, enc_resp->encoder_id,
                                    DRM_MODE_OBJECT_ENCODER);
         if (!obj) {
-                ret = -EINVAL;
+                ret = -ENOENT;
                 goto out;
         }
         encoder = obj_to_encoder(obj);
@@ -2141,7 +2141,7 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
                                    DRM_MODE_OBJECT_CRTC);
         if (!obj) {
                 DRM_DEBUG_KMS("Unknown CRTC ID %d\n", crtc_req->crtc_id);
-                ret = -EINVAL;
+                ret = -ENOENT;
                 goto out;
         }
         crtc = obj_to_crtc(obj);
@@ -2164,7 +2164,7 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
                         if (!fb) {
                                 DRM_DEBUG_KMS("Unknown FB ID%d\n",
                                                 crtc_req->fb_id);
-                                ret = -EINVAL;
+                                ret = -ENOENT;
                                 goto out;
                         }
                 }
@@ -2232,7 +2232,7 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
                         if (!obj) {
                                 DRM_DEBUG_KMS("Connector id %d unknown\n",
                                                 out_id);
-                                ret = -EINVAL;
+                                ret = -ENOENT;
                                 goto out;
                         }
                         connector = obj_to_connector(obj);
@@ -2280,7 +2280,7 @@ static int drm_mode_cursor_common(struct drm_device *dev,
         obj = drm_mode_object_find(dev, req->crtc_id, DRM_MODE_OBJECT_CRTC);
         if (!obj) {
                 DRM_DEBUG_KMS("Unknown CRTC ID %d\n", req->crtc_id);
-                return -EINVAL;
+                return -ENOENT;
         }
         crtc = obj_to_crtc(obj);
 
@@ -2489,6 +2489,8 @@ static int format_check(const struct drm_mode_fb_cmd2 *r)
         case DRM_FORMAT_YVU444:
                 return 0;
         default:
+                DRM_DEBUG_KMS("invalid pixel format %s\n",
+                              drm_get_format_name(r->pixel_format));
                 return -EINVAL;
         }
 }
@@ -2654,7 +2656,7 @@ fail_lookup:
         mutex_unlock(&dev->mode_config.fb_lock);
         mutex_unlock(&file_priv->fbs_lock);
 
-        return -EINVAL;
+        return -ENOENT;
 }
 
 /**
@@ -2682,7 +2684,7 @@ int drm_mode_getfb(struct drm_device *dev,
 
         fb = drm_framebuffer_lookup(dev, r->fb_id);
         if (!fb)
-                return -EINVAL;
+                return -ENOENT;
 
         r->height = fb->height;
         r->width = fb->width;
@@ -2727,7 +2729,7 @@ int drm_mode_dirtyfb_ioctl(struct drm_device *dev,
 
         fb = drm_framebuffer_lookup(dev, r->fb_id);
         if (!fb)
-                return -EINVAL;
+                return -ENOENT;
 
         num_clips = r->num_clips;
         clips_ptr = (struct drm_clip_rect __user *)(unsigned long)r->clips_ptr;
@@ -3059,7 +3061,7 @@ int drm_mode_getproperty_ioctl(struct drm_device *dev,
         drm_modeset_lock_all(dev);
         obj = drm_mode_object_find(dev, out_resp->prop_id, DRM_MODE_OBJECT_PROPERTY);
         if (!obj) {
-                ret = -EINVAL;
+                ret = -ENOENT;
                 goto done;
         }
         property = obj_to_property(obj);
@@ -3188,7 +3190,7 @@ int drm_mode_getblob_ioctl(struct drm_device *dev,
         drm_modeset_lock_all(dev);
         obj = drm_mode_object_find(dev, out_resp->blob_id, DRM_MODE_OBJECT_BLOB);
         if (!obj) {
-                ret = -EINVAL;
+                ret = -ENOENT;
                 goto done;
         }
         blob = obj_to_blob(obj);
@@ -3349,7 +3351,7 @@ int drm_mode_obj_get_properties_ioctl(struct drm_device *dev, void *data,
 
         obj = drm_mode_object_find(dev, arg->obj_id, arg->obj_type);
         if (!obj) {
-                ret = -EINVAL;
+                ret = -ENOENT;
                 goto out;
         }
         if (!obj->properties) {
@@ -3402,8 +3404,10 @@ int drm_mode_obj_set_property_ioctl(struct drm_device *dev, void *data,
         drm_modeset_lock_all(dev);
 
         arg_obj = drm_mode_object_find(dev, arg->obj_id, arg->obj_type);
-        if (!arg_obj)
+        if (!arg_obj) {
+                ret = -ENOENT;
                 goto out;
+        }
         if (!arg_obj->properties)
                 goto out;
 
@@ -3416,8 +3420,10 @@ int drm_mode_obj_set_property_ioctl(struct drm_device *dev, void *data,
 
         prop_obj = drm_mode_object_find(dev, arg->prop_id,
                                         DRM_MODE_OBJECT_PROPERTY);
-        if (!prop_obj)
+        if (!prop_obj) {
+                ret = -ENOENT;
                 goto out;
+        }
         property = obj_to_property(prop_obj);
 
         if (!drm_property_change_is_valid(property, arg->value))
@@ -3502,7 +3508,7 @@ int drm_mode_gamma_set_ioctl(struct drm_device *dev,
         drm_modeset_lock_all(dev);
         obj = drm_mode_object_find(dev, crtc_lut->crtc_id, DRM_MODE_OBJECT_CRTC);
         if (!obj) {
-                ret = -EINVAL;
+                ret = -ENOENT;
                 goto out;
         }
         crtc = obj_to_crtc(obj);
@@ -3561,7 +3567,7 @@ int drm_mode_gamma_get_ioctl(struct drm_device *dev,
         drm_modeset_lock_all(dev);
         obj = drm_mode_object_find(dev, crtc_lut->crtc_id, DRM_MODE_OBJECT_CRTC);
         if (!obj) {
-                ret = -EINVAL;
+                ret = -ENOENT;
                 goto out;
         }
         crtc = obj_to_crtc(obj);
@@ -3615,7 +3621,7 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
 
         obj = drm_mode_object_find(dev, page_flip->crtc_id, DRM_MODE_OBJECT_CRTC);
         if (!obj)
-                return -EINVAL;
+                return -ENOENT;
         crtc = obj_to_crtc(obj);
 
         mutex_lock(&crtc->mutex);
@@ -3632,8 +3638,10 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
                 goto out;
 
         fb = drm_framebuffer_lookup(dev, page_flip->fb_id);
-        if (!fb)
+        if (!fb) {
+                ret = -ENOENT;
                 goto out;
+        }
 
         ret = drm_crtc_check_viewport(crtc, crtc->x, crtc->y, &crtc->mode, fb);
         if (ret)
@@ -3822,7 +3830,8 @@ void drm_fb_get_bpp_depth(uint32_t format, unsigned int *depth,
                 *bpp = 32;
                 break;
         default:
-                DRM_DEBUG_KMS("unsupported pixel format\n");
+                DRM_DEBUG_KMS("unsupported pixel format %s\n",
+                              drm_get_format_name(format));
                 *depth = 0;
                 *bpp = 0;
                 break;
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index 0d6469d74be4..01361aba033b 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -405,22 +405,25 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
                               struct drm_framebuffer *old_fb)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_display_mode *adjusted_mode, saved_mode, saved_hwmode;
+        struct drm_display_mode *adjusted_mode, saved_mode;
         struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
         struct drm_encoder_helper_funcs *encoder_funcs;
         int saved_x, saved_y;
+        bool saved_enabled;
         struct drm_encoder *encoder;
         bool ret = true;
 
+        saved_enabled = crtc->enabled;
         crtc->enabled = drm_helper_crtc_in_use(crtc);
         if (!crtc->enabled)
                 return true;
 
         adjusted_mode = drm_mode_duplicate(dev, mode);
-        if (!adjusted_mode)
+        if (!adjusted_mode) {
+                crtc->enabled = saved_enabled;
                 return false;
+        }
 
-        saved_hwmode = crtc->hwmode;
         saved_mode = crtc->mode;
         saved_x = crtc->x;
         saved_y = crtc->y;
@@ -539,7 +542,7 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
 done:
         drm_mode_destroy(dev, adjusted_mode);
         if (!ret) {
-                crtc->hwmode = saved_hwmode;
+                crtc->enabled = saved_enabled;
                 crtc->mode = saved_mode;
                 crtc->x = saved_x;
                 crtc->y = saved_y;
@@ -567,6 +570,14 @@ drm_crtc_helper_disable(struct drm_crtc *crtc)
                                 continue;
 
                         connector->encoder = NULL;
+
+                        /*
+                         * drm_helper_disable_unused_functions() ought to be
+                         * doing this, but since we've decoupled the encoder
+                         * from the connector above, the required connection
+                         * between them is henceforth no longer available.
+                         */
+                        connector->dpms = DRM_MODE_DPMS_OFF;
                 }
         }
 
@@ -593,9 +604,8 @@ drm_crtc_helper_disable(struct drm_crtc *crtc)
 int drm_crtc_helper_set_config(struct drm_mode_set *set)
 {
         struct drm_device *dev;
-        struct drm_crtc *save_crtcs, *new_crtc, *crtc;
+        struct drm_crtc *new_crtc;
         struct drm_encoder *save_encoders, *new_encoder, *encoder;
-        struct drm_framebuffer *old_fb = NULL;
         bool mode_changed = false; /* if true do a full mode set */
         bool fb_changed = false; /* if true and !mode_changed just do a flip */
         struct drm_connector *save_connectors, *connector;
@@ -631,38 +641,28 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
 
         dev = set->crtc->dev;
 
-        /* Allocate space for the backup of all (non-pointer) crtc, encoder and
-         * connector data. */
-        save_crtcs = kzalloc(dev->mode_config.num_crtc *
-                             sizeof(struct drm_crtc), GFP_KERNEL);
-        if (!save_crtcs)
-                return -ENOMEM;
-
+        /*
+         * Allocate space for the backup of all (non-pointer) encoder and
+         * connector data.
+         */
         save_encoders = kzalloc(dev->mode_config.num_encoder *
                                 sizeof(struct drm_encoder), GFP_KERNEL);
-        if (!save_encoders) {
-                kfree(save_crtcs);
+        if (!save_encoders)
                 return -ENOMEM;
-        }
 
         save_connectors = kzalloc(dev->mode_config.num_connector *
                                 sizeof(struct drm_connector), GFP_KERNEL);
         if (!save_connectors) {
-                kfree(save_crtcs);
                 kfree(save_encoders);
                 return -ENOMEM;
         }
 
-        /* Copy data. Note that driver private data is not affected.
+        /*
+         * Copy data. Note that driver private data is not affected.
          * Should anything bad happen only the expected state is
          * restored, not the drivers personal bookkeeping.
          */
         count = 0;
-        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-                save_crtcs[count++] = *crtc;
-        }
-
-        count = 0;
         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
                 save_encoders[count++] = *encoder;
         }
@@ -785,19 +785,17 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                 mode_changed = true;
 
         if (mode_changed) {
-                set->crtc->enabled = drm_helper_crtc_in_use(set->crtc);
-                if (set->crtc->enabled) {
+                if (drm_helper_crtc_in_use(set->crtc)) {
                         DRM_DEBUG_KMS("attempting to set mode from"
                                         " userspace\n");
                         drm_mode_debug_printmodeline(set->mode);
-                        old_fb = set->crtc->fb;
                         set->crtc->fb = set->fb;
                         if (!drm_crtc_helper_set_mode(set->crtc, set->mode,
                                                       set->x, set->y,
-                                                      old_fb)) {
+                                                      save_set.fb)) {
                                 DRM_ERROR("failed to set mode on [CRTC:%d]\n",
                                           set->crtc->base.id);
-                                set->crtc->fb = old_fb;
+                                set->crtc->fb = save_set.fb;
                                 ret = -EINVAL;
                                 goto fail;
                         }
@@ -812,31 +810,24 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
         } else if (fb_changed) {
                 set->crtc->x = set->x;
                 set->crtc->y = set->y;
-
-                old_fb = set->crtc->fb;
-                if (set->crtc->fb != set->fb)
-                        set->crtc->fb = set->fb;
+                set->crtc->fb = set->fb;
                 ret = crtc_funcs->mode_set_base(set->crtc,
-                                                set->x, set->y, old_fb);
+                                                set->x, set->y, save_set.fb);
                 if (ret != 0) {
-                        set->crtc->fb = old_fb;
+                        set->crtc->x = save_set.x;
+                        set->crtc->y = save_set.y;
+                        set->crtc->fb = save_set.fb;
                         goto fail;
                 }
         }
 
         kfree(save_connectors);
         kfree(save_encoders);
-        kfree(save_crtcs);
         return 0;
 
 fail:
         /* Restore all previous data. */
         count = 0;
-        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-                *crtc = save_crtcs[count++];
-        }
-
-        count = 0;
         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
                 *encoder = save_encoders[count++];
         }
@@ -854,7 +845,6 @@ fail:
 
         kfree(save_connectors);
         kfree(save_encoders);
-        kfree(save_crtcs);
         return ret;
 }
 EXPORT_SYMBOL(drm_crtc_helper_set_config);
@@ -1135,14 +1125,14 @@ void drm_kms_helper_poll_fini(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_kms_helper_poll_fini);
 
-void drm_helper_hpd_irq_event(struct drm_device *dev)
+bool drm_helper_hpd_irq_event(struct drm_device *dev)
 {
         struct drm_connector *connector;
         enum drm_connector_status old_status;
         bool changed = false;
 
         if (!dev->mode_config.poll_enabled)
-                return;
+                return false;
 
         mutex_lock(&dev->mode_config.mutex);
         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
@@ -1167,5 +1157,7 @@ void drm_helper_hpd_irq_event(struct drm_device *dev)
 
         if (changed)
                 drm_kms_helper_hotplug_event(dev);
+
+        return changed;
 }
 EXPORT_SYMBOL(drm_helper_hpd_irq_event);
diff --git a/drivers/gpu/drm/drm_debugfs.c b/drivers/gpu/drm/drm_debugfs.c
index a05087cf846d..b4b51d46f339 100644
--- a/drivers/gpu/drm/drm_debugfs.c
+++ b/drivers/gpu/drm/drm_debugfs.c
@@ -42,7 +42,7 @@
  * Initialization, etc.
  **************************************************/
 
-static struct drm_info_list drm_debugfs_list[] = {
+static const struct drm_info_list drm_debugfs_list[] = {
         {"name", drm_name_info, 0},
         {"vm", drm_vm_info, 0},
         {"clients", drm_clients_info, 0},
@@ -84,7 +84,7 @@ static const struct file_operations drm_debugfs_fops = {
  * Create a given set of debugfs files represented by an array of
  * gdm_debugfs_lists in the given root directory.
  */
-int drm_debugfs_create_files(struct drm_info_list *files, int count,
+int drm_debugfs_create_files(const struct drm_info_list *files, int count,
                              struct dentry *root, struct drm_minor *minor)
 {
         struct drm_device *dev = minor->dev;
@@ -188,7 +188,7 @@ int drm_debugfs_init(struct drm_minor *minor, int minor_id,
  *
  * Remove all debugfs entries created by debugfs_init().
  */
-int drm_debugfs_remove_files(struct drm_info_list *files, int count,
+int drm_debugfs_remove_files(const struct drm_info_list *files, int count,
                              struct drm_minor *minor)
 {
         struct list_head *pos, *q;
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 05e197d32c45..d9137e49c4e8 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -403,7 +403,7 @@ long drm_ioctl(struct file *filp,
 
       err_i1:
         if (!ioctl)
-                DRM_DEBUG("invalid iotcl: pid=%d, dev=0x%lx, auth=%d, cmd=0x%02x, nr=0x%02x\n",
+                DRM_DEBUG("invalid ioctl: pid=%d, dev=0x%lx, auth=%d, cmd=0x%02x, nr=0x%02x\n",
                           task_pid_nr(current),
                           (long)old_encode_dev(file_priv->minor->device),
                           file_priv->authenticated, cmd, nr);
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index f1764ec5818b..fb7cf0e796f6 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -458,6 +458,15 @@ static const struct drm_display_mode drm_dmt_modes[] = {
                    DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 };
 
+/*
+ * These more or less come from the DMT spec.  The 720x400 modes are
+ * inferred from historical 80x25 practice.  The 640x480@67 and 832x624@75
+ * modes are old-school Mac modes.  The EDID spec says the 1152x864@75 mode
+ * should be 1152x870, again for the Mac, but instead we use the x864 DMT
+ * mode.
+ *
+ * The DMT modes have been fact-checked; the rest are mild guesses.
+ */
 static const struct drm_display_mode edid_est_modes[] = {
         { DRM_MODE("800x600", DRM_MODE_TYPE_DRIVER, 40000, 800, 840,
                    968, 1056, 0, 600, 601, 605, 628, 0,
@@ -560,7 +569,7 @@ static const struct minimode est3_modes[] = {
         { 1600, 1200, 75, 0 },
         { 1600, 1200, 85, 0 },
         { 1792, 1344, 60, 0 },
-        { 1792, 1344, 85, 0 },
+        { 1792, 1344, 75, 0 },
         { 1856, 1392, 60, 0 },
         { 1856, 1392, 75, 0 },
         { 1920, 1200, 60, 1 },
@@ -1320,7 +1329,7 @@ static u32 edid_get_quirks(struct edid *edid)
 }
 
 #define MODE_SIZE(m) ((m)->hdisplay * (m)->vdisplay)
-#define MODE_REFRESH_DIFF(m,r) (abs((m)->vrefresh - target_refresh))
+#define MODE_REFRESH_DIFF(c,t) (abs((c) - (t)))
 
 /**
  * edid_fixup_preferred - set preferred modes based on quirk list
@@ -1335,6 +1344,7 @@ static void edid_fixup_preferred(struct drm_connector *connector,
 {
         struct drm_display_mode *t, *cur_mode, *preferred_mode;
         int target_refresh = 0;
+        int cur_vrefresh, preferred_vrefresh;
 
         if (list_empty(&connector->probed_modes))
                 return;
@@ -1357,10 +1367,14 @@ static void edid_fixup_preferred(struct drm_connector *connector,
                 if (MODE_SIZE(cur_mode) > MODE_SIZE(preferred_mode))
                         preferred_mode = cur_mode;
 
+                cur_vrefresh = cur_mode->vrefresh ?
+                        cur_mode->vrefresh : drm_mode_vrefresh(cur_mode);
+                preferred_vrefresh = preferred_mode->vrefresh ?
+                        preferred_mode->vrefresh : drm_mode_vrefresh(preferred_mode);
                 /* At a given size, try to get closest to target refresh */
                 if ((MODE_SIZE(cur_mode) == MODE_SIZE(preferred_mode)) &&
-                    MODE_REFRESH_DIFF(cur_mode, target_refresh) <
-                    MODE_REFRESH_DIFF(preferred_mode, target_refresh)) {
+                    MODE_REFRESH_DIFF(cur_vrefresh, target_refresh) <
+                    MODE_REFRESH_DIFF(preferred_vrefresh, target_refresh)) {
                         preferred_mode = cur_mode;
                 }
         }
@@ -2080,7 +2094,7 @@ drm_est3_modes(struct drm_connector *connector, struct detailed_timing *timing)
         u8 *est = ((u8 *)timing) + 5;
 
         for (i = 0; i < 6; i++) {
-                for (j = 7; j > 0; j--) {
+                for (j = 7; j >= 0; j--) {
                         m = (i * 8) + (7 - j);
                         if (m >= ARRAY_SIZE(est3_modes))
                                 break;
@@ -3473,6 +3487,19 @@ int drm_add_modes_noedid(struct drm_connector *connector,
 }
 EXPORT_SYMBOL(drm_add_modes_noedid);
 
+void drm_set_preferred_mode(struct drm_connector *connector,
+                           int hpref, int vpref)
+{
+        struct drm_display_mode *mode;
+
+        list_for_each_entry(mode, &connector->probed_modes, head) {
+                if (drm_mode_width(mode)  == hpref &&
+                    drm_mode_height(mode) == vpref)
+                        mode->type |= DRM_MODE_TYPE_PREFERRED;
+        }
+}
+EXPORT_SYMBOL(drm_set_preferred_mode);
+
 /**
  * drm_hdmi_avi_infoframe_from_display_mode() - fill an HDMI AVI infoframe with
  *                                              data from a DRM display mode
diff --git a/drivers/gpu/drm/drm_fops.c b/drivers/gpu/drm/drm_fops.c
index d0e27667a4eb..3a7176ce2540 100644
--- a/drivers/gpu/drm/drm_fops.c
+++ b/drivers/gpu/drm/drm_fops.c
@@ -239,7 +239,8 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
 
         priv->ioctl_count = 0;
         /* for compatibility root is always authenticated */
-        priv->authenticated = capable(CAP_SYS_ADMIN);
+        priv->always_authenticated = capable(CAP_SYS_ADMIN);
+        priv->authenticated = priv->always_authenticated;
         priv->lock_count = 0;
 
         INIT_LIST_HEAD(&priv->lhead);
@@ -378,8 +379,10 @@ static void drm_events_release(struct drm_file *file_priv)
                 }
 
         /* Remove unconsumed events */
-        list_for_each_entry_safe(e, et, &file_priv->event_list, link)
+        list_for_each_entry_safe(e, et, &file_priv->event_list, link) {
+                list_del(&e->link);
                 e->destroy(e);
+        }
 
         spin_unlock_irqrestore(&dev->event_lock, flags);
 }
@@ -531,7 +534,7 @@ int drm_release(struct inode *inode, struct file *filp)
                 list_for_each_entry(temp, &dev->filelist, lhead) {
                         if ((temp->master == file_priv->master) &&
                             (temp != file_priv))
-                                temp->authenticated = 0;
+                                temp->authenticated = temp->always_authenticated;
                 }
 
                 /**
diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c
index f9af048828ea..d80d95289e10 100644
--- a/drivers/gpu/drm/drm_irq.c
+++ b/drivers/gpu/drm/drm_irq.c
@@ -219,7 +219,7 @@ int drm_vblank_init(struct drm_device *dev, int num_crtcs)
         for (i = 0; i < num_crtcs; i++)
                 init_waitqueue_head(&dev->vblank[i].queue);
 
-        DRM_INFO("Supports vblank timestamp caching Rev 1 (10.10.2010).\n");
+        DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
 
         /* Driver specific high-precision vblank timestamping supported? */
         if (dev->driver->get_vblank_timestamp)
@@ -586,24 +586,20 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc,
          * code gets preempted or delayed for some reason.
          */
         for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
-                /* Disable preemption to make it very likely to
-                 * succeed in the first iteration even on PREEMPT_RT kernel.
+                /*
+                 * Get vertical and horizontal scanout position vpos, hpos,
+                 * and bounding timestamps stime, etime, pre/post query.
                  */
-                preempt_disable();
+                vbl_status = dev->driver->get_scanout_position(dev, crtc, &vpos,
+                                                               &hpos, &stime, &etime);
 
-                /* Get system timestamp before query. */
-                stime = ktime_get();
-
-                /* Get vertical and horizontal scanout pos. vpos, hpos. */
-                vbl_status = dev->driver->get_scanout_position(dev, crtc, &vpos, &hpos);
-
-                /* Get system timestamp after query. */
-                etime = ktime_get();
+                /*
+                 * Get correction for CLOCK_MONOTONIC -> CLOCK_REALTIME if
+                 * CLOCK_REALTIME is requested.
+                 */
                 if (!drm_timestamp_monotonic)
                         mono_time_offset = ktime_get_monotonic_offset();
 
-                preempt_enable();
-
                 /* Return as no-op if scanout query unsupported or failed. */
                 if (!(vbl_status & DRM_SCANOUTPOS_VALID)) {
                         DRM_DEBUG("crtc %d : scanoutpos query failed [%d].\n",
@@ -611,6 +607,7 @@ int drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc,
                         return -EIO;
                 }
 
+                /* Compute uncertainty in timestamp of scanout position query. */
                 duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
 
                 /* Accept result with <  max_error nsecs timing uncertainty. */
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index b0733153dfd2..85071a1c4547 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -1041,7 +1041,7 @@ void drm_mode_connector_list_update(struct drm_connector *connector)
                                 /* if equal delete the probed mode */
                                 mode->status = pmode->status;
                                 /* Merge type bits together */
-                                mode->type |= pmode->type;
+                                mode->type = pmode->type;
                                 list_del(&pmode->head);
                                 drm_mode_destroy(connector->dev, pmode);
                                 break;
diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c
index f00d7a9671ea..02679793c9e2 100644
--- a/drivers/gpu/drm/drm_pci.c
+++ b/drivers/gpu/drm/drm_pci.c
@@ -80,7 +80,7 @@ drm_dma_handle_t *drm_pci_alloc(struct drm_device * dev, size_t size, size_t ali
         /* Reserve */
         for (addr = (unsigned long)dmah->vaddr, sz = size;
              sz > 0; addr += PAGE_SIZE, sz -= PAGE_SIZE) {
-                SetPageReserved(virt_to_page(addr));
+                SetPageReserved(virt_to_page((void *)addr));
         }
 
         return dmah;
@@ -103,7 +103,7 @@ void __drm_pci_free(struct drm_device * dev, drm_dma_handle_t * dmah)
                 /* Unreserve */
                 for (addr = (unsigned long)dmah->vaddr, sz = dmah->size;
                      sz > 0; addr += PAGE_SIZE, sz -= PAGE_SIZE) {
-                        ClearPageReserved(virt_to_page(addr));
+                        ClearPageReserved(virt_to_page((void *)addr));
                 }
                 dma_free_coherent(&dev->pdev->dev, dmah->size, dmah->vaddr,
                                   dmah->busaddr);
diff --git a/drivers/gpu/drm/drm_stub.c b/drivers/gpu/drm/drm_stub.c
index 26055abf94ee..c200136a5d8e 100644
--- a/drivers/gpu/drm/drm_stub.c
+++ b/drivers/gpu/drm/drm_stub.c
@@ -255,16 +255,20 @@ int drm_dropmaster_ioctl(struct drm_device *dev, void *data,
 }
 
 /**
- * Get a secondary minor number.
+ * drm_get_minor - Allocate and register new DRM minor
+ * @dev: DRM device
+ * @minor: Pointer to where new minor is stored
+ * @type: Type of minor
  *
- * \param dev device data structure
- * \param sec-minor structure to hold the assigned minor
- * \return negative number on failure.
+ * Allocate a new minor of the given type and register it. A pointer to the new
+ * minor is returned in @minor.
+ * Caller must hold the global DRM mutex.
  *
- * Search an empty entry and initialize it to the given parameters. This
- * routines assigns minor numbers to secondary heads of multi-headed cards
+ * RETURNS:
+ * 0 on success, negative error code on failure.
  */
-int drm_get_minor(struct drm_device *dev, struct drm_minor **minor, int type)
+static int drm_get_minor(struct drm_device *dev, struct drm_minor **minor,
+                         int type)
 {
         struct drm_minor *new_minor;
         int ret;
@@ -321,37 +325,48 @@ err_idr:
         *minor = NULL;
         return ret;
 }
-EXPORT_SYMBOL(drm_get_minor);
 
 /**
- * Put a secondary minor number.
+ * drm_unplug_minor - Unplug DRM minor
+ * @minor: Minor to unplug
  *
- * \param sec_minor - structure to be released
- * \return always zero
+ * Unplugs the given DRM minor but keeps the object. So after this returns,
+ * minor->dev is still valid so existing open-files can still access it to get
+ * device information from their drm_file ojects.
+ * If the minor is already unplugged or if @minor is NULL, nothing is done.
+ * The global DRM mutex must be held by the caller.
  */
-int drm_put_minor(struct drm_minor **minor_p)
+static void drm_unplug_minor(struct drm_minor *minor)
 {
-        struct drm_minor *minor = *minor_p;
-
-        DRM_DEBUG("release secondary minor %d\n", minor->index);
+        if (!minor || !device_is_registered(minor->kdev))
+                return;
 
 #if defined(CONFIG_DEBUG_FS)
         drm_debugfs_cleanup(minor);
 #endif
 
         drm_sysfs_device_remove(minor);
-
         idr_remove(&drm_minors_idr, minor->index);
-
-        kfree(minor);
-        *minor_p = NULL;
-        return 0;
 }
-EXPORT_SYMBOL(drm_put_minor);
 
-static void drm_unplug_minor(struct drm_minor *minor)
+/**
+ * drm_put_minor - Destroy DRM minor
+ * @minor: Minor to destroy
+ *
+ * This calls drm_unplug_minor() on the given minor and then frees it. Nothing
+ * is done if @minor is NULL. It is fine to call this on already unplugged
+ * minors.
+ * The global DRM mutex must be held by the caller.
+ */
+static void drm_put_minor(struct drm_minor *minor)
 {
-        drm_sysfs_device_remove(minor);
+        if (!minor)
+                return;
+
+        DRM_DEBUG("release secondary minor %d\n", minor->index);
+
+        drm_unplug_minor(minor);
+        kfree(minor);
 }
 
 /**
@@ -472,6 +487,10 @@ EXPORT_SYMBOL(drm_dev_alloc);
  */
 void drm_dev_free(struct drm_device *dev)
 {
+        drm_put_minor(dev->control);
+        drm_put_minor(dev->render);
+        drm_put_minor(dev->primary);
+
         if (dev->driver->driver_features & DRIVER_GEM)
                 drm_gem_destroy(dev);
 
@@ -547,13 +566,11 @@ err_unload:
         if (dev->driver->unload)
                 dev->driver->unload(dev);
 err_primary_node:
-        drm_put_minor(&dev->primary);
+        drm_put_minor(dev->primary);
 err_render_node:
-        if (dev->render)
-                drm_put_minor(&dev->render);
+        drm_put_minor(dev->render);
 err_control_node:
-        if (dev->control)
-                drm_put_minor(&dev->control);
+        drm_put_minor(dev->control);
 err_agp:
         if (dev->driver->bus->agp_destroy)
                 dev->driver->bus->agp_destroy(dev);
@@ -588,11 +605,9 @@ void drm_dev_unregister(struct drm_device *dev)
         list_for_each_entry_safe(r_list, list_temp, &dev->maplist, head)
                 drm_rmmap(dev, r_list->map);
 
-        if (dev->control)
-                drm_put_minor(&dev->control);
-        if (dev->render)
-                drm_put_minor(&dev->render);
-        drm_put_minor(&dev->primary);
+        drm_unplug_minor(dev->control);
+        drm_unplug_minor(dev->render);
+        drm_unplug_minor(dev->primary);
 
         list_del(&dev->driver_item);
 }
diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c
index 2290b3b73832..1a35ea53106b 100644
--- a/drivers/gpu/drm/drm_sysfs.c
+++ b/drivers/gpu/drm/drm_sysfs.c
@@ -22,8 +22,8 @@
 #include <drm/drm_core.h>
 #include <drm/drmP.h>
 
-#define to_drm_minor(d) container_of(d, struct drm_minor, kdev)
-#define to_drm_connector(d) container_of(d, struct drm_connector, kdev)
+#define to_drm_minor(d) dev_get_drvdata(d)
+#define to_drm_connector(d) dev_get_drvdata(d)
 
 static struct device_type drm_sysfs_device_minor = {
         .name = "drm_minor"
@@ -162,20 +162,6 @@ void drm_sysfs_destroy(void)
         drm_class = NULL;
 }
 
-/**
- * drm_sysfs_device_release - do nothing
- * @dev: Linux device
- *
- * Normally, this would free the DRM device associated with @dev, along
- * with cleaning up any other stuff.  But we do that in the DRM core, so
- * this function can just return and hope that the core does its job.
- */
-static void drm_sysfs_device_release(struct device *dev)
-{
-        memset(dev, 0, sizeof(struct device));
-        return;
-}
-
 /*
  * Connector properties
  */
@@ -380,11 +366,6 @@ static struct bin_attribute edid_attr = {
  * properties (so far, connection status, dpms, mode list & edid) and
  * generate a hotplug event so userspace knows there's a new connector
  * available.
- *
- * Note:
- * This routine should only be called *once* for each registered connector.
- * A second call for an already registered connector will trigger the BUG_ON
- * below.
  */
 int drm_sysfs_connector_add(struct drm_connector *connector)
 {
@@ -394,29 +375,25 @@ int drm_sysfs_connector_add(struct drm_connector *connector)
         int i;
         int ret;
 
-        /* We shouldn't get called more than once for the same connector */
-        BUG_ON(device_is_registered(&connector->kdev));
-
-        connector->kdev.parent = &dev->primary->kdev;
-        connector->kdev.class = drm_class;
-        connector->kdev.release = drm_sysfs_device_release;
+        if (connector->kdev)
+                return 0;
 
+        connector->kdev = device_create(drm_class, dev->primary->kdev,
+                                        0, connector, "card%d-%s",
+                                        dev->primary->index, drm_get_connector_name(connector));
         DRM_DEBUG("adding \"%s\" to sysfs\n",
                   drm_get_connector_name(connector));
 
-        dev_set_name(&connector->kdev, "card%d-%s",
-                     dev->primary->index, drm_get_connector_name(connector));
-        ret = device_register(&connector->kdev);
-
-        if (ret) {
-                DRM_ERROR("failed to register connector device: %d\n", ret);
+        if (IS_ERR(connector->kdev)) {
+                DRM_ERROR("failed to register connector device: %ld\n", PTR_ERR(connector->kdev));
+                ret = PTR_ERR(connector->kdev);
                 goto out;
         }
 
         /* Standard attributes */
 
         for (attr_cnt = 0; attr_cnt < ARRAY_SIZE(connector_attrs); attr_cnt++) {
-                ret = device_create_file(&connector->kdev, &connector_attrs[attr_cnt]);
+                ret = device_create_file(connector->kdev, &connector_attrs[attr_cnt]);
                 if (ret)
                         goto err_out_files;
         }
@@ -433,7 +410,7 @@ int drm_sysfs_connector_add(struct drm_connector *connector)
                 case DRM_MODE_CONNECTOR_Component:
                 case DRM_MODE_CONNECTOR_TV:
                         for (opt_cnt = 0; opt_cnt < ARRAY_SIZE(connector_attrs_opt1); opt_cnt++) {
-                                ret = device_create_file(&connector->kdev, &connector_attrs_opt1[opt_cnt]);
+                                ret = device_create_file(connector->kdev, &connector_attrs_opt1[opt_cnt]);
                                 if (ret)
                                         goto err_out_files;
                         }
@@ -442,7 +419,7 @@ int drm_sysfs_connector_add(struct drm_connector *connector)
                         break;
         }
 
-        ret = sysfs_create_bin_file(&connector->kdev.kobj, &edid_attr);
+        ret = sysfs_create_bin_file(&connector->kdev->kobj, &edid_attr);
         if (ret)
                 goto err_out_files;
 
@@ -453,10 +430,10 @@ int drm_sysfs_connector_add(struct drm_connector *connector)
 
 err_out_files:
         for (i = 0; i < opt_cnt; i++)
-                device_remove_file(&connector->kdev, &connector_attrs_opt1[i]);
+                device_remove_file(connector->kdev, &connector_attrs_opt1[i]);
         for (i = 0; i < attr_cnt; i++)
-                device_remove_file(&connector->kdev, &connector_attrs[i]);
-        device_unregister(&connector->kdev);
+                device_remove_file(connector->kdev, &connector_attrs[i]);
+        device_unregister(connector->kdev);
 
 out:
         return ret;
@@ -480,16 +457,16 @@ void drm_sysfs_connector_remove(struct drm_connector *connector)
 {
         int i;
 
-        if (!connector->kdev.parent)
+        if (!connector->kdev)
                 return;
         DRM_DEBUG("removing \"%s\" from sysfs\n",
                   drm_get_connector_name(connector));
 
         for (i = 0; i < ARRAY_SIZE(connector_attrs); i++)
-                device_remove_file(&connector->kdev, &connector_attrs[i]);
-        sysfs_remove_bin_file(&connector->kdev.kobj, &edid_attr);
-        device_unregister(&connector->kdev);
-        connector->kdev.parent = NULL;
+                device_remove_file(connector->kdev, &connector_attrs[i]);
+        sysfs_remove_bin_file(&connector->kdev->kobj, &edid_attr);
+        device_unregister(connector->kdev);
+        connector->kdev = NULL;
 }
 EXPORT_SYMBOL(drm_sysfs_connector_remove);
 
@@ -508,7 +485,7 @@ void drm_sysfs_hotplug_event(struct drm_device *dev)
 
         DRM_DEBUG("generating hotplug event\n");
 
-        kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, envp);
+        kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, envp);
 }
 EXPORT_SYMBOL(drm_sysfs_hotplug_event);
 
@@ -523,15 +500,8 @@ EXPORT_SYMBOL(drm_sysfs_hotplug_event);
  */
 int drm_sysfs_device_add(struct drm_minor *minor)
 {
-        int err;
         char *minor_str;
 
-        minor->kdev.parent = minor->dev->dev;
-
-        minor->kdev.class = drm_class;
-        minor->kdev.release = drm_sysfs_device_release;
-        minor->kdev.devt = minor->device;
-        minor->kdev.type = &drm_sysfs_device_minor;
         if (minor->type == DRM_MINOR_CONTROL)
                 minor_str = "controlD%d";
         else if (minor->type == DRM_MINOR_RENDER)
@@ -539,18 +509,14 @@ int drm_sysfs_device_add(struct drm_minor *minor)
         else
                 minor_str = "card%d";
 
-        dev_set_name(&minor->kdev, minor_str, minor->index);
-
-        err = device_register(&minor->kdev);
-        if (err) {
-                DRM_ERROR("device add failed: %d\n", err);
-                goto err_out;
+        minor->kdev = device_create(drm_class, minor->dev->dev,
+                                    MKDEV(DRM_MAJOR, minor->index),
+                                    minor, minor_str, minor->index);
+        if (IS_ERR(minor->kdev)) {
+                DRM_ERROR("device create failed %ld\n", PTR_ERR(minor->kdev));
+                return PTR_ERR(minor->kdev);
         }
-
         return 0;
-
-err_out:
-        return err;
 }
 
 /**
@@ -562,9 +528,9 @@ err_out:
  */
 void drm_sysfs_device_remove(struct drm_minor *minor)
 {
-        if (minor->kdev.parent)
-                device_unregister(&minor->kdev);
-        minor->kdev.parent = NULL;
+        if (minor->kdev)
+                device_destroy(drm_class, MKDEV(DRM_MAJOR, minor->index));
+        minor->kdev = NULL;
 }
 
 
diff --git a/drivers/gpu/drm/drm_vm.c b/drivers/gpu/drm/drm_vm.c
index b5c5af7328df..93e95d7efd57 100644
--- a/drivers/gpu/drm/drm_vm.c
+++ b/drivers/gpu/drm/drm_vm.c
@@ -301,7 +301,7 @@ static int drm_do_vm_dma_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 
         offset = (unsigned long)vmf->virtual_address - vma->vm_start;   /* vm_[pg]off[set] should be 0 */
         page_nr = offset >> PAGE_SHIFT; /* page_nr could just be vmf->pgoff */
-        page = virt_to_page((dma->pagelist[page_nr] + (offset & (~PAGE_MASK))));
+        page = virt_to_page((void *)dma->pagelist[page_nr]);
 
         get_page(page);
         vmf->page = page;
diff --git a/drivers/gpu/drm/gma500/cdv_device.c b/drivers/gpu/drm/gma500/cdv_device.c
index 162f686c532d..5a9a6a3063a8 100644
--- a/drivers/gpu/drm/gma500/cdv_device.c
+++ b/drivers/gpu/drm/gma500/cdv_device.c
@@ -634,6 +634,7 @@ const struct psb_ops cdv_chip_ops = {
         .crtcs = 2,
         .hdmi_mask = (1 << 0) | (1 << 1),
         .lvds_mask = (1 << 1),
+        .sdvo_mask = (1 << 0),
         .cursor_needs_phys = 0,
         .sgx_offset = MRST_SGX_OFFSET,
         .chip_setup = cdv_chip_setup,
diff --git a/drivers/gpu/drm/gma500/cdv_intel_dp.c b/drivers/gpu/drm/gma500/cdv_intel_dp.c
index f4eb43573cad..f88a1815d87c 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_dp.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_dp.c
@@ -666,7 +666,7 @@ cdv_intel_dp_i2c_init(struct gma_connector *connector,
         strncpy (intel_dp->adapter.name, name, sizeof(intel_dp->adapter.name) - 1);
         intel_dp->adapter.name[sizeof(intel_dp->adapter.name) - 1] = '\0';
         intel_dp->adapter.algo_data = &intel_dp->algo;
-        intel_dp->adapter.dev.parent = &connector->base.kdev;
+        intel_dp->adapter.dev.parent = connector->base.kdev;
 
         if (is_edp(encoder))
                 cdv_intel_edp_panel_vdd_on(encoder);
diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c
index 01dd7d225762..94b3fec22c28 100644
--- a/drivers/gpu/drm/gma500/framebuffer.c
+++ b/drivers/gpu/drm/gma500/framebuffer.c
@@ -714,7 +714,7 @@ static void psb_setup_outputs(struct drm_device *dev)
                         clone_mask = (1 << INTEL_OUTPUT_ANALOG);
                         break;
                 case INTEL_OUTPUT_SDVO:
-                        crtc_mask = ((1 << 0) | (1 << 1));
+                        crtc_mask = dev_priv->ops->sdvo_mask;
                         clone_mask = (1 << INTEL_OUTPUT_SDVO);
                         break;
                 case INTEL_OUTPUT_LVDS:
diff --git a/drivers/gpu/drm/gma500/intel_gmbus.c b/drivers/gpu/drm/gma500/intel_gmbus.c
index 62cd42e88f28..566d330aaeea 100644
--- a/drivers/gpu/drm/gma500/intel_gmbus.c
+++ b/drivers/gpu/drm/gma500/intel_gmbus.c
@@ -51,6 +51,9 @@
 #define wait_for(COND, MS) _wait_for(COND, MS, 1)
 #define wait_for_atomic(COND, MS) _wait_for(COND, MS, 0)
 
+#define GMBUS_REG_READ(reg) ioread32(dev_priv->gmbus_reg + (reg))
+#define GMBUS_REG_WRITE(reg, val) iowrite32((val), dev_priv->gmbus_reg + (reg))
+
 /* Intel GPIO access functions */
 
 #define I2C_RISEFALL_TIME 20
@@ -71,7 +74,8 @@ struct intel_gpio {
 void
 gma_intel_i2c_reset(struct drm_device *dev)
 {
-        REG_WRITE(GMBUS0, 0);
+        struct drm_psb_private *dev_priv = dev->dev_private;
+        GMBUS_REG_WRITE(GMBUS0, 0);
 }
 
 static void intel_i2c_quirk_set(struct drm_psb_private *dev_priv, bool enable)
@@ -98,11 +102,10 @@ static void intel_i2c_quirk_set(struct drm_psb_private *dev_priv, bool enable)
 static u32 get_reserved(struct intel_gpio *gpio)
 {
         struct drm_psb_private *dev_priv = gpio->dev_priv;
-        struct drm_device *dev = dev_priv->dev;
         u32 reserved = 0;
 
         /* On most chips, these bits must be preserved in software. */
-        reserved = REG_READ(gpio->reg) &
+        reserved = GMBUS_REG_READ(gpio->reg) &
                                      (GPIO_DATA_PULLUP_DISABLE |
                                       GPIO_CLOCK_PULLUP_DISABLE);
 
@@ -113,29 +116,26 @@ static int get_clock(void *data)
 {
         struct intel_gpio *gpio = data;
         struct drm_psb_private *dev_priv = gpio->dev_priv;
-        struct drm_device *dev = dev_priv->dev;
         u32 reserved = get_reserved(gpio);
-        REG_WRITE(gpio->reg, reserved | GPIO_CLOCK_DIR_MASK);
-        REG_WRITE(gpio->reg, reserved);
-        return (REG_READ(gpio->reg) & GPIO_CLOCK_VAL_IN) != 0;
+        GMBUS_REG_WRITE(gpio->reg, reserved | GPIO_CLOCK_DIR_MASK);
+        GMBUS_REG_WRITE(gpio->reg, reserved);
+        return (GMBUS_REG_READ(gpio->reg) & GPIO_CLOCK_VAL_IN) != 0;
 }
 
 static int get_data(void *data)
 {
         struct intel_gpio *gpio = data;
         struct drm_psb_private *dev_priv = gpio->dev_priv;
-        struct drm_device *dev = dev_priv->dev;
         u32 reserved = get_reserved(gpio);
-        REG_WRITE(gpio->reg, reserved | GPIO_DATA_DIR_MASK);
-        REG_WRITE(gpio->reg, reserved);
-        return (REG_READ(gpio->reg) & GPIO_DATA_VAL_IN) != 0;
+        GMBUS_REG_WRITE(gpio->reg, reserved | GPIO_DATA_DIR_MASK);
+        GMBUS_REG_WRITE(gpio->reg, reserved);
+        return (GMBUS_REG_READ(gpio->reg) & GPIO_DATA_VAL_IN) != 0;
 }
 
 static void set_clock(void *data, int state_high)
 {
         struct intel_gpio *gpio = data;
         struct drm_psb_private *dev_priv = gpio->dev_priv;
-        struct drm_device *dev = dev_priv->dev;
         u32 reserved = get_reserved(gpio);
         u32 clock_bits;
 
@@ -145,15 +145,14 @@ static void set_clock(void *data, int state_high)
                 clock_bits = GPIO_CLOCK_DIR_OUT | GPIO_CLOCK_DIR_MASK |
                         GPIO_CLOCK_VAL_MASK;
 
-        REG_WRITE(gpio->reg, reserved | clock_bits);
-        REG_READ(gpio->reg); /* Posting */
+        GMBUS_REG_WRITE(gpio->reg, reserved | clock_bits);
+        GMBUS_REG_READ(gpio->reg); /* Posting */
 }
 
 static void set_data(void *data, int state_high)
 {
         struct intel_gpio *gpio = data;
         struct drm_psb_private *dev_priv = gpio->dev_priv;
-        struct drm_device *dev = dev_priv->dev;
         u32 reserved = get_reserved(gpio);
         u32 data_bits;
 
@@ -163,8 +162,8 @@ static void set_data(void *data, int state_high)
                 data_bits = GPIO_DATA_DIR_OUT | GPIO_DATA_DIR_MASK |
                         GPIO_DATA_VAL_MASK;
 
-        REG_WRITE(gpio->reg, reserved | data_bits);
-        REG_READ(gpio->reg);
+        GMBUS_REG_WRITE(gpio->reg, reserved | data_bits);
+        GMBUS_REG_READ(gpio->reg);
 }
 
 static struct i2c_adapter *
@@ -251,7 +250,6 @@ gmbus_xfer(struct i2c_adapter *adapter,
                                                struct intel_gmbus,
                                                adapter);
         struct drm_psb_private *dev_priv = adapter->algo_data;
-        struct drm_device *dev = dev_priv->dev;
         int i, reg_offset;
 
         if (bus->force_bit)
@@ -260,28 +258,30 @@ gmbus_xfer(struct i2c_adapter *adapter,
 
         reg_offset = 0;
 
-        REG_WRITE(GMBUS0 + reg_offset, bus->reg0);
+        GMBUS_REG_WRITE(GMBUS0 + reg_offset, bus->reg0);
 
         for (i = 0; i < num; i++) {
                 u16 len = msgs[i].len;
                 u8 *buf = msgs[i].buf;
 
                 if (msgs[i].flags & I2C_M_RD) {
-                        REG_WRITE(GMBUS1 + reg_offset,
-                                   GMBUS_CYCLE_WAIT | (i + 1 == num ? GMBUS_CYCLE_STOP : 0) |
-                                   (len << GMBUS_BYTE_COUNT_SHIFT) |
-                                   (msgs[i].addr << GMBUS_SLAVE_ADDR_SHIFT) |
-                                   GMBUS_SLAVE_READ | GMBUS_SW_RDY);
-                        REG_READ(GMBUS2+reg_offset);
+                        GMBUS_REG_WRITE(GMBUS1 + reg_offset,
+                                        GMBUS_CYCLE_WAIT |
+                                        (i + 1 == num ? GMBUS_CYCLE_STOP : 0) |
+                                        (len << GMBUS_BYTE_COUNT_SHIFT) |
+                                        (msgs[i].addr << GMBUS_SLAVE_ADDR_SHIFT) |
+                                        GMBUS_SLAVE_READ | GMBUS_SW_RDY);
+                        GMBUS_REG_READ(GMBUS2+reg_offset);
                         do {
                                 u32 val, loop = 0;
 
-                                if (wait_for(REG_READ(GMBUS2 + reg_offset) & (GMBUS_SATOER | GMBUS_HW_RDY), 50))
+                                if (wait_for(GMBUS_REG_READ(GMBUS2 + reg_offset) &
+                                             (GMBUS_SATOER | GMBUS_HW_RDY), 50))
                                         goto timeout;
-                                if (REG_READ(GMBUS2 + reg_offset) & GMBUS_SATOER)
+                                if (GMBUS_REG_READ(GMBUS2 + reg_offset) & GMBUS_SATOER)
                                         goto clear_err;
 
-                                val = REG_READ(GMBUS3 + reg_offset);
+                                val = GMBUS_REG_READ(GMBUS3 + reg_offset);
                                 do {
                                         *buf++ = val & 0xff;
                                         val >>= 8;
@@ -295,18 +295,20 @@ gmbus_xfer(struct i2c_adapter *adapter,
                                 val |= *buf++ << (8 * loop);
                         } while (--len && ++loop < 4);
 
-                        REG_WRITE(GMBUS3 + reg_offset, val);
-                        REG_WRITE(GMBUS1 + reg_offset,
+                        GMBUS_REG_WRITE(GMBUS3 + reg_offset, val);
+                        GMBUS_REG_WRITE(GMBUS1 + reg_offset,
                                    (i + 1 == num ? GMBUS_CYCLE_STOP : GMBUS_CYCLE_WAIT) |
                                    (msgs[i].len << GMBUS_BYTE_COUNT_SHIFT) |
                                    (msgs[i].addr << GMBUS_SLAVE_ADDR_SHIFT) |
                                    GMBUS_SLAVE_WRITE | GMBUS_SW_RDY);
-                        REG_READ(GMBUS2+reg_offset);
+                        GMBUS_REG_READ(GMBUS2+reg_offset);
 
                         while (len) {
-                                if (wait_for(REG_READ(GMBUS2 + reg_offset) & (GMBUS_SATOER | GMBUS_HW_RDY), 50))
+                                if (wait_for(GMBUS_REG_READ(GMBUS2 + reg_offset) &
+                                             (GMBUS_SATOER | GMBUS_HW_RDY), 50))
                                         goto timeout;
-                                if (REG_READ(GMBUS2 + reg_offset) & GMBUS_SATOER)
+                                if (GMBUS_REG_READ(GMBUS2 + reg_offset) &
+                                    GMBUS_SATOER)
                                         goto clear_err;
 
                                 val = loop = 0;
@@ -314,14 +316,14 @@ gmbus_xfer(struct i2c_adapter *adapter,
                                         val |= *buf++ << (8 * loop);
                                 } while (--len && ++loop < 4);
 
-                                REG_WRITE(GMBUS3 + reg_offset, val);
-                                REG_READ(GMBUS2+reg_offset);
+                                GMBUS_REG_WRITE(GMBUS3 + reg_offset, val);
+                                GMBUS_REG_READ(GMBUS2+reg_offset);
                         }
                 }
 
-                if (i + 1 < num && wait_for(REG_READ(GMBUS2 + reg_offset) & (GMBUS_SATOER | GMBUS_HW_WAIT_PHASE), 50))
+                if (i + 1 < num && wait_for(GMBUS_REG_READ(GMBUS2 + reg_offset) & (GMBUS_SATOER | GMBUS_HW_WAIT_PHASE), 50))
                         goto timeout;
-                if (REG_READ(GMBUS2 + reg_offset) & GMBUS_SATOER)
+                if (GMBUS_REG_READ(GMBUS2 + reg_offset) & GMBUS_SATOER)
                         goto clear_err;
         }
 
@@ -332,20 +334,20 @@ clear_err:
          * of resetting the GMBUS controller and so clearing the
          * BUS_ERROR raised by the slave's NAK.
          */
-        REG_WRITE(GMBUS1 + reg_offset, GMBUS_SW_CLR_INT);
-        REG_WRITE(GMBUS1 + reg_offset, 0);
+        GMBUS_REG_WRITE(GMBUS1 + reg_offset, GMBUS_SW_CLR_INT);
+        GMBUS_REG_WRITE(GMBUS1 + reg_offset, 0);
 
 done:
         /* Mark the GMBUS interface as disabled. We will re-enable it at the
          * start of the next xfer, till then let it sleep.
          */
-        REG_WRITE(GMBUS0 + reg_offset, 0);
+        GMBUS_REG_WRITE(GMBUS0 + reg_offset, 0);
         return i;
 
 timeout:
         DRM_INFO("GMBUS timed out, falling back to bit banging on pin %d [%s]\n",
                  bus->reg0 & 0xff, bus->adapter.name);
-        REG_WRITE(GMBUS0 + reg_offset, 0);
+        GMBUS_REG_WRITE(GMBUS0 + reg_offset, 0);
 
         /* Hardware may not support GMBUS over these pins? Try GPIO bitbanging instead. */
         bus->force_bit = intel_gpio_create(dev_priv, bus->reg0 & 0xff);
@@ -399,6 +401,11 @@ int gma_intel_setup_gmbus(struct drm_device *dev)
         if (dev_priv->gmbus == NULL)
                 return -ENOMEM;
 
+        if (IS_MRST(dev))
+                dev_priv->gmbus_reg = dev_priv->aux_reg;
+        else
+                dev_priv->gmbus_reg = dev_priv->vdc_reg;
+
         for (i = 0; i < GMBUS_NUM_PORTS; i++) {
                 struct intel_gmbus *bus = &dev_priv->gmbus[i];
 
@@ -487,6 +494,7 @@ void gma_intel_teardown_gmbus(struct drm_device *dev)
                 i2c_del_adapter(&bus->adapter);
         }
 
+        dev_priv->gmbus_reg = NULL; /* iounmap is done in driver_unload */
         kfree(dev_priv->gmbus);
         dev_priv->gmbus = NULL;
 }
diff --git a/drivers/gpu/drm/gma500/oaktrail_crtc.c b/drivers/gpu/drm/gma500/oaktrail_crtc.c
index 54c98962b73e..8195e8592107 100644
--- a/drivers/gpu/drm/gma500/oaktrail_crtc.c
+++ b/drivers/gpu/drm/gma500/oaktrail_crtc.c
@@ -26,24 +26,10 @@
 #include "gma_display.h"
 #include "power.h"
 
-struct psb_intel_range_t {
-        int min, max;
-};
-
-struct oaktrail_limit_t {
-        struct psb_intel_range_t dot, m, p1;
-};
-
-struct oaktrail_clock_t {
-        /* derived values */
-        int dot;
-        int m;
-        int p1;
-};
-
-#define MRST_LIMIT_LVDS_100L        0
-#define MRST_LIMIT_LVDS_83          1
-#define MRST_LIMIT_LVDS_100         2
+#define MRST_LIMIT_LVDS_100L    0
+#define MRST_LIMIT_LVDS_83      1
+#define MRST_LIMIT_LVDS_100     2
+#define MRST_LIMIT_SDVO         3
 
 #define MRST_DOT_MIN              19750
 #define MRST_DOT_MAX              120000
@@ -57,21 +43,40 @@ struct oaktrail_clock_t {
 #define MRST_P1_MAX_0               7
 #define MRST_P1_MAX_1               8
 
-static const struct oaktrail_limit_t oaktrail_limits[] = {
+static bool mrst_lvds_find_best_pll(const struct gma_limit_t *limit,
+                                    struct drm_crtc *crtc, int target,
+                                    int refclk, struct gma_clock_t *best_clock);
+
+static bool mrst_sdvo_find_best_pll(const struct gma_limit_t *limit,
+                                    struct drm_crtc *crtc, int target,
+                                    int refclk, struct gma_clock_t *best_clock);
+
+static const struct gma_limit_t mrst_limits[] = {
         {                       /* MRST_LIMIT_LVDS_100L */
          .dot = {.min = MRST_DOT_MIN, .max = MRST_DOT_MAX},
          .m = {.min = MRST_M_MIN_100L, .max = MRST_M_MAX_100L},
          .p1 = {.min = MRST_P1_MIN, .max = MRST_P1_MAX_1},
+         .find_pll = mrst_lvds_find_best_pll,
          },
         {                       /* MRST_LIMIT_LVDS_83L */
          .dot = {.min = MRST_DOT_MIN, .max = MRST_DOT_MAX},
          .m = {.min = MRST_M_MIN_83, .max = MRST_M_MAX_83},
          .p1 = {.min = MRST_P1_MIN, .max = MRST_P1_MAX_0},
+         .find_pll = mrst_lvds_find_best_pll,
          },
         {                       /* MRST_LIMIT_LVDS_100 */
          .dot = {.min = MRST_DOT_MIN, .max = MRST_DOT_MAX},
          .m = {.min = MRST_M_MIN_100, .max = MRST_M_MAX_100},
          .p1 = {.min = MRST_P1_MIN, .max = MRST_P1_MAX_1},
+         .find_pll = mrst_lvds_find_best_pll,
+         },
+        {                       /* MRST_LIMIT_SDVO */
+         .vco = {.min = 1400000, .max = 2800000},
+         .n = {.min = 3, .max = 7},
+         .m = {.min = 80, .max = 137},
+         .p1 = {.min = 1, .max = 2},
+         .p2 = {.dot_limit = 200000, .p2_slow = 10, .p2_fast = 10},
+         .find_pll = mrst_sdvo_find_best_pll,
          },
 };
 
@@ -82,9 +87,10 @@ static const u32 oaktrail_m_converts[] = {
         0x12, 0x09, 0x24, 0x32, 0x39, 0x1c,
 };
 
-static const struct oaktrail_limit_t *oaktrail_limit(struct drm_crtc *crtc)
+static const struct gma_limit_t *mrst_limit(struct drm_crtc *crtc,
+                                            int refclk)
 {
-        const struct oaktrail_limit_t *limit = NULL;
+        const struct gma_limit_t *limit = NULL;
         struct drm_device *dev = crtc->dev;
         struct drm_psb_private *dev_priv = dev->dev_private;
 
@@ -92,45 +98,100 @@ static const struct oaktrail_limit_t *oaktrail_limit(struct drm_crtc *crtc)
             || gma_pipe_has_type(crtc, INTEL_OUTPUT_MIPI)) {
                 switch (dev_priv->core_freq) {
                 case 100:
-                        limit = &oaktrail_limits[MRST_LIMIT_LVDS_100L];
+                        limit = &mrst_limits[MRST_LIMIT_LVDS_100L];
                         break;
                 case 166:
-                        limit = &oaktrail_limits[MRST_LIMIT_LVDS_83];
+                        limit = &mrst_limits[MRST_LIMIT_LVDS_83];
                         break;
                 case 200:
-                        limit = &oaktrail_limits[MRST_LIMIT_LVDS_100];
+                        limit = &mrst_limits[MRST_LIMIT_LVDS_100];
                         break;
                 }
+        } else if (gma_pipe_has_type(crtc, INTEL_OUTPUT_SDVO)) {
+                limit = &mrst_limits[MRST_LIMIT_SDVO];
         } else {
                 limit = NULL;
-                dev_err(dev->dev, "oaktrail_limit Wrong display type.\n");
+                dev_err(dev->dev, "mrst_limit Wrong display type.\n");
         }
 
         return limit;
 }
 
 /** Derive the pixel clock for the given refclk and divisors for 8xx chips. */
-static void oaktrail_clock(int refclk, struct oaktrail_clock_t *clock)
+static void mrst_lvds_clock(int refclk, struct gma_clock_t *clock)
 {
         clock->dot = (refclk * clock->m) / (14 * clock->p1);
 }
 
-static void mrstPrintPll(char *prefix, struct oaktrail_clock_t *clock)
+static void mrst_print_pll(struct gma_clock_t *clock)
 {
-        pr_debug("%s: dotclock = %d,  m = %d, p1 = %d.\n",
-             prefix, clock->dot, clock->m, clock->p1);
+        DRM_DEBUG_DRIVER("dotclock=%d,  m=%d, m1=%d, m2=%d, n=%d, p1=%d, p2=%d\n",
+                         clock->dot, clock->m, clock->m1, clock->m2, clock->n,
+                         clock->p1, clock->p2);
+}
+
+static bool mrst_sdvo_find_best_pll(const struct gma_limit_t *limit,
+                                    struct drm_crtc *crtc, int target,
+                                    int refclk, struct gma_clock_t *best_clock)
+{
+        struct gma_clock_t clock;
+        u32 target_vco, actual_freq;
+        s32 freq_error, min_error = 100000;
+
+        memset(best_clock, 0, sizeof(*best_clock));
+
+        for (clock.m = limit->m.min; clock.m <= limit->m.max; clock.m++) {
+                for (clock.n = limit->n.min; clock.n <= limit->n.max;
+                     clock.n++) {
+                        for (clock.p1 = limit->p1.min;
+                             clock.p1 <= limit->p1.max; clock.p1++) {
+                                /* p2 value always stored in p2_slow on SDVO */
+                                clock.p = clock.p1 * limit->p2.p2_slow;
+                                target_vco = target * clock.p;
+
+                                /* VCO will increase at this point so break */
+                                if (target_vco > limit->vco.max)
+                                        break;
+
+                                if (target_vco < limit->vco.min)
+                                        continue;
+
+                                actual_freq = (refclk * clock.m) /
+                                              (clock.n * clock.p);
+                                freq_error = 10000 -
+                                             ((target * 10000) / actual_freq);
+
+                                if (freq_error < -min_error) {
+                                        /* freq_error will start to decrease at
+                                           this point so break */
+                                        break;
+                                }
+
+                                if (freq_error < 0)
+                                        freq_error = -freq_error;
+
+                                if (freq_error < min_error) {
+                                        min_error = freq_error;
+                                        *best_clock = clock;
+                                }
+                        }
+                }
+                if (min_error == 0)
+                        break;
+        }
+
+        return min_error == 0;
 }
 
 /**
  * Returns a set of divisors for the desired target clock with the given refclk,
  * or FALSE.  Divisor values are the actual divisors for
  */
-static bool
-mrstFindBestPLL(struct drm_crtc *crtc, int target, int refclk,
-                struct oaktrail_clock_t *best_clock)
+static bool mrst_lvds_find_best_pll(const struct gma_limit_t *limit,
+                                    struct drm_crtc *crtc, int target,
+                                    int refclk, struct gma_clock_t *best_clock)
 {
-        struct oaktrail_clock_t clock;
-        const struct oaktrail_limit_t *limit = oaktrail_limit(crtc);
+        struct gma_clock_t clock;
         int err = target;
 
         memset(best_clock, 0, sizeof(*best_clock));
@@ -140,7 +201,7 @@ mrstFindBestPLL(struct drm_crtc *crtc, int target, int refclk,
                      clock.p1++) {
                         int this_err;
 
-                        oaktrail_clock(refclk, &clock);
+                        mrst_lvds_clock(refclk, &clock);
 
                         this_err = abs(clock.dot - target);
                         if (this_err < err) {
@@ -149,7 +210,6 @@ mrstFindBestPLL(struct drm_crtc *crtc, int target, int refclk,
                         }
                 }
         }
-        dev_dbg(crtc->dev->dev, "mrstFindBestPLL err = %d.\n", err);
         return err != target;
 }
 
@@ -167,8 +227,10 @@ static void oaktrail_crtc_dpms(struct drm_crtc *crtc, int mode)
         int pipe = gma_crtc->pipe;
         const struct psb_offset *map = &dev_priv->regmap[pipe];
         u32 temp;
+        int i;
+        int need_aux = gma_pipe_has_type(crtc, INTEL_OUTPUT_SDVO) ? 1 : 0;
 
-        if (pipe == 1) {
+        if (gma_pipe_has_type(crtc, INTEL_OUTPUT_HDMI)) {
                 oaktrail_crtc_hdmi_dpms(crtc, mode);
                 return;
         }
@@ -183,35 +245,45 @@ static void oaktrail_crtc_dpms(struct drm_crtc *crtc, int mode)
         case DRM_MODE_DPMS_ON:
         case DRM_MODE_DPMS_STANDBY:
         case DRM_MODE_DPMS_SUSPEND:
-                /* Enable the DPLL */
-                temp = REG_READ(map->dpll);
-                if ((temp & DPLL_VCO_ENABLE) == 0) {
-                        REG_WRITE(map->dpll, temp);
-                        REG_READ(map->dpll);
-                        /* Wait for the clocks to stabilize. */
-                        udelay(150);
-                        REG_WRITE(map->dpll, temp | DPLL_VCO_ENABLE);
-                        REG_READ(map->dpll);
-                        /* Wait for the clocks to stabilize. */
-                        udelay(150);
-                        REG_WRITE(map->dpll, temp | DPLL_VCO_ENABLE);
-                        REG_READ(map->dpll);
-                        /* Wait for the clocks to stabilize. */
-                        udelay(150);
-                }
-                /* Enable the pipe */
-                temp = REG_READ(map->conf);
-                if ((temp & PIPEACONF_ENABLE) == 0)
-                        REG_WRITE(map->conf, temp | PIPEACONF_ENABLE);
-                /* Enable the plane */
-                temp = REG_READ(map->cntr);
-                if ((temp & DISPLAY_PLANE_ENABLE) == 0) {
-                        REG_WRITE(map->cntr,
-                                  temp | DISPLAY_PLANE_ENABLE);
-                        /* Flush the plane changes */
-                        REG_WRITE(map->base, REG_READ(map->base));
-                }
+                for (i = 0; i <= need_aux; i++) {
+                        /* Enable the DPLL */
+                        temp = REG_READ_WITH_AUX(map->dpll, i);
+                        if ((temp & DPLL_VCO_ENABLE) == 0) {
+                                REG_WRITE_WITH_AUX(map->dpll, temp, i);
+                                REG_READ_WITH_AUX(map->dpll, i);
+                                /* Wait for the clocks to stabilize. */
+                                udelay(150);
+                                REG_WRITE_WITH_AUX(map->dpll,
+                                                   temp | DPLL_VCO_ENABLE, i);
+                                REG_READ_WITH_AUX(map->dpll, i);
+                                /* Wait for the clocks to stabilize. */
+                                udelay(150);
+                                REG_WRITE_WITH_AUX(map->dpll,
+                                                   temp | DPLL_VCO_ENABLE, i);
+                                REG_READ_WITH_AUX(map->dpll, i);
+                                /* Wait for the clocks to stabilize. */
+                                udelay(150);
+                        }
+
+                        /* Enable the pipe */
+                        temp = REG_READ_WITH_AUX(map->conf, i);
+                        if ((temp & PIPEACONF_ENABLE) == 0) {
+                                REG_WRITE_WITH_AUX(map->conf,
+                                                   temp | PIPEACONF_ENABLE, i);
+                        }
+
+                        /* Enable the plane */
+                        temp = REG_READ_WITH_AUX(map->cntr, i);
+                        if ((temp & DISPLAY_PLANE_ENABLE) == 0) {
+                                REG_WRITE_WITH_AUX(map->cntr,
+                                                   temp | DISPLAY_PLANE_ENABLE,
+                                                   i);
+                                /* Flush the plane changes */
+                                REG_WRITE_WITH_AUX(map->base,
+                                        REG_READ_WITH_AUX(map->base, i), i);
+                        }
 
+                }
                 gma_crtc_load_lut(crtc);
 
                 /* Give the overlay scaler a chance to enable
@@ -223,48 +295,52 @@ static void oaktrail_crtc_dpms(struct drm_crtc *crtc, int mode)
                  * if it's on this pipe */
                 /* psb_intel_crtc_dpms_video(crtc, FALSE); TODO */
 
-                /* Disable the VGA plane that we never use */
-                REG_WRITE(VGACNTRL, VGA_DISP_DISABLE);
-                /* Disable display plane */
-                temp = REG_READ(map->cntr);
-                if ((temp & DISPLAY_PLANE_ENABLE) != 0) {
-                        REG_WRITE(map->cntr,
-                                  temp & ~DISPLAY_PLANE_ENABLE);
-                        /* Flush the plane changes */
-                        REG_WRITE(map->base, REG_READ(map->base));
-                        REG_READ(map->base);
-                }
+                for (i = 0; i <= need_aux; i++) {
+                        /* Disable the VGA plane that we never use */
+                        REG_WRITE_WITH_AUX(VGACNTRL, VGA_DISP_DISABLE, i);
+                        /* Disable display plane */
+                        temp = REG_READ_WITH_AUX(map->cntr, i);
+                        if ((temp & DISPLAY_PLANE_ENABLE) != 0) {
+                                REG_WRITE_WITH_AUX(map->cntr,
+                                        temp & ~DISPLAY_PLANE_ENABLE, i);
+                                /* Flush the plane changes */
+                                REG_WRITE_WITH_AUX(map->base,
+                                                   REG_READ(map->base), i);
+                                REG_READ_WITH_AUX(map->base, i);
+                        }
 
-                /* Next, disable display pipes */
-                temp = REG_READ(map->conf);
-                if ((temp & PIPEACONF_ENABLE) != 0) {
-                        REG_WRITE(map->conf, temp & ~PIPEACONF_ENABLE);
-                        REG_READ(map->conf);
-                }
-                /* Wait for for the pipe disable to take effect. */
-                gma_wait_for_vblank(dev);
+                        /* Next, disable display pipes */
+                        temp = REG_READ_WITH_AUX(map->conf, i);
+                        if ((temp & PIPEACONF_ENABLE) != 0) {
+                                REG_WRITE_WITH_AUX(map->conf,
+                                                   temp & ~PIPEACONF_ENABLE, i);
+                                REG_READ_WITH_AUX(map->conf, i);
+                        }
+                        /* Wait for for the pipe disable to take effect. */
+                        gma_wait_for_vblank(dev);
+
+                        temp = REG_READ_WITH_AUX(map->dpll, i);
+                        if ((temp & DPLL_VCO_ENABLE) != 0) {
+                                REG_WRITE_WITH_AUX(map->dpll,
+                                                   temp & ~DPLL_VCO_ENABLE, i);
+                                REG_READ_WITH_AUX(map->dpll, i);
+                        }
 
-                temp = REG_READ(map->dpll);
-                if ((temp & DPLL_VCO_ENABLE) != 0) {
-                        REG_WRITE(map->dpll, temp & ~DPLL_VCO_ENABLE);
-                        REG_READ(map->dpll);
+                        /* Wait for the clocks to turn off. */
+                        udelay(150);
                 }
-
-                /* Wait for the clocks to turn off. */
-                udelay(150);
                 break;
         }
 
-        /*Set FIFO Watermarks*/
-        REG_WRITE(DSPARB, 0x3FFF);
-        REG_WRITE(DSPFW1, 0x3F88080A);
-        REG_WRITE(DSPFW2, 0x0b060808);
+        /* Set FIFO Watermarks (values taken from EMGD) */
+        REG_WRITE(DSPARB, 0x3f80);
+        REG_WRITE(DSPFW1, 0x3f8f0404);
+        REG_WRITE(DSPFW2, 0x04040f04);
         REG_WRITE(DSPFW3, 0x0);
-        REG_WRITE(DSPFW4, 0x08030404);
+        REG_WRITE(DSPFW4, 0x04040404);
         REG_WRITE(DSPFW5, 0x04040404);
         REG_WRITE(DSPFW6, 0x78);
-        REG_WRITE(0x70400, REG_READ(0x70400) | 0x4000);
-        /* Must write Bit 14 of the Chicken Bit Register */
+        REG_WRITE(DSPCHICKENBIT, REG_READ(DSPCHICKENBIT) | 0xc040);
 
         gma_power_end(dev);
 }
@@ -297,7 +373,8 @@ static int oaktrail_crtc_mode_set(struct drm_crtc *crtc,
         int pipe = gma_crtc->pipe;
         const struct psb_offset *map = &dev_priv->regmap[pipe];
         int refclk = 0;
-        struct oaktrail_clock_t clock;
+        struct gma_clock_t clock;
+        const struct gma_limit_t *limit;
         u32 dpll = 0, fp = 0, dspcntr, pipeconf;
         bool ok, is_sdvo = false;
         bool is_lvds = false;
@@ -306,8 +383,10 @@ static int oaktrail_crtc_mode_set(struct drm_crtc *crtc,
         struct gma_encoder *gma_encoder = NULL;
         uint64_t scalingType = DRM_MODE_SCALE_FULLSCREEN;
         struct drm_connector *connector;
+        int i;
+        int need_aux = gma_pipe_has_type(crtc, INTEL_OUTPUT_SDVO) ? 1 : 0;
 
-        if (pipe == 1)
+        if (gma_pipe_has_type(crtc, INTEL_OUTPUT_HDMI))
                 return oaktrail_crtc_hdmi_mode_set(crtc, mode, adjusted_mode, x, y, old_fb);
 
         if (!gma_power_begin(dev, true))
@@ -340,15 +419,17 @@ static int oaktrail_crtc_mode_set(struct drm_crtc *crtc,
         }
 
         /* Disable the VGA plane that we never use */
-        REG_WRITE(VGACNTRL, VGA_DISP_DISABLE);
+        for (i = 0; i <= need_aux; i++)
+                REG_WRITE_WITH_AUX(VGACNTRL, VGA_DISP_DISABLE, i);
 
         /* Disable the panel fitter if it was on our pipe */
         if (oaktrail_panel_fitter_pipe(dev) == pipe)
                 REG_WRITE(PFIT_CONTROL, 0);
 
-        REG_WRITE(map->src,
-                  ((mode->crtc_hdisplay - 1) << 16) |
-                  (mode->crtc_vdisplay - 1));
+        for (i = 0; i <= need_aux; i++) {
+                REG_WRITE_WITH_AUX(map->src, ((mode->crtc_hdisplay - 1) << 16) |
+                                             (mode->crtc_vdisplay - 1), i);
+        }
 
         if (gma_encoder)
                 drm_object_property_get_value(&connector->base,
@@ -365,35 +446,39 @@ static int oaktrail_crtc_mode_set(struct drm_crtc *crtc,
                 offsetY = (adjusted_mode->crtc_vdisplay -
                            mode->crtc_vdisplay) / 2;
 
-                REG_WRITE(map->htotal, (mode->crtc_hdisplay - 1) |
-                        ((adjusted_mode->crtc_htotal - 1) << 16));
-                REG_WRITE(map->vtotal, (mode->crtc_vdisplay - 1) |
-                        ((adjusted_mode->crtc_vtotal - 1) << 16));
-                REG_WRITE(map->hblank,
-                        (adjusted_mode->crtc_hblank_start - offsetX - 1) |
-                        ((adjusted_mode->crtc_hblank_end - offsetX - 1) << 16));
-                REG_WRITE(map->hsync,
-                        (adjusted_mode->crtc_hsync_start - offsetX - 1) |
-                        ((adjusted_mode->crtc_hsync_end - offsetX - 1) << 16));
-                REG_WRITE(map->vblank,
-                        (adjusted_mode->crtc_vblank_start - offsetY - 1) |
-                        ((adjusted_mode->crtc_vblank_end - offsetY - 1) << 16));
-                REG_WRITE(map->vsync,
-                        (adjusted_mode->crtc_vsync_start - offsetY - 1) |
-                        ((adjusted_mode->crtc_vsync_end - offsetY - 1) << 16));
+                for (i = 0; i <= need_aux; i++) {
+                        REG_WRITE_WITH_AUX(map->htotal, (mode->crtc_hdisplay - 1) |
+                                ((adjusted_mode->crtc_htotal - 1) << 16), i);
+                        REG_WRITE_WITH_AUX(map->vtotal, (mode->crtc_vdisplay - 1) |
+                                ((adjusted_mode->crtc_vtotal - 1) << 16), i);
+                        REG_WRITE_WITH_AUX(map->hblank,
+                                (adjusted_mode->crtc_hblank_start - offsetX - 1) |
+                                ((adjusted_mode->crtc_hblank_end - offsetX - 1) << 16), i);
+                        REG_WRITE_WITH_AUX(map->hsync,
+                                (adjusted_mode->crtc_hsync_start - offsetX - 1) |
+                                ((adjusted_mode->crtc_hsync_end - offsetX - 1) << 16), i);
+                        REG_WRITE_WITH_AUX(map->vblank,
+                                (adjusted_mode->crtc_vblank_start - offsetY - 1) |
+                                ((adjusted_mode->crtc_vblank_end - offsetY - 1) << 16), i);
+                        REG_WRITE_WITH_AUX(map->vsync,
+                                (adjusted_mode->crtc_vsync_start - offsetY - 1) |
+                                ((adjusted_mode->crtc_vsync_end - offsetY - 1) << 16), i);
+                }
         } else {
-                REG_WRITE(map->htotal, (adjusted_mode->crtc_hdisplay - 1) |
-                        ((adjusted_mode->crtc_htotal - 1) << 16));
-                REG_WRITE(map->vtotal, (adjusted_mode->crtc_vdisplay - 1) |
-                        ((adjusted_mode->crtc_vtotal - 1) << 16));
-                REG_WRITE(map->hblank, (adjusted_mode->crtc_hblank_start - 1) |
-                        ((adjusted_mode->crtc_hblank_end - 1) << 16));
-                REG_WRITE(map->hsync, (adjusted_mode->crtc_hsync_start - 1) |
-                        ((adjusted_mode->crtc_hsync_end - 1) << 16));
-                REG_WRITE(map->vblank, (adjusted_mode->crtc_vblank_start - 1) |
-                        ((adjusted_mode->crtc_vblank_end - 1) << 16));
-                REG_WRITE(map->vsync, (adjusted_mode->crtc_vsync_start - 1) |
-                        ((adjusted_mode->crtc_vsync_end - 1) << 16));
+                for (i = 0; i <= need_aux; i++) {
+                        REG_WRITE_WITH_AUX(map->htotal, (adjusted_mode->crtc_hdisplay - 1) |
+                                ((adjusted_mode->crtc_htotal - 1) << 16), i);
+                        REG_WRITE_WITH_AUX(map->vtotal, (adjusted_mode->crtc_vdisplay - 1) |
+                                ((adjusted_mode->crtc_vtotal - 1) << 16), i);
+                        REG_WRITE_WITH_AUX(map->hblank, (adjusted_mode->crtc_hblank_start - 1) |
+                                ((adjusted_mode->crtc_hblank_end - 1) << 16), i);
+                        REG_WRITE_WITH_AUX(map->hsync, (adjusted_mode->crtc_hsync_start - 1) |
+                                ((adjusted_mode->crtc_hsync_end - 1) << 16), i);
+                        REG_WRITE_WITH_AUX(map->vblank, (adjusted_mode->crtc_vblank_start - 1) |
+                                ((adjusted_mode->crtc_vblank_end - 1) << 16), i);
+                        REG_WRITE_WITH_AUX(map->vsync, (adjusted_mode->crtc_vsync_start - 1) |
+                                ((adjusted_mode->crtc_vsync_end - 1) << 16), i);
+                }
         }
 
         /* Flush the plane changes */
@@ -418,21 +503,30 @@ static int oaktrail_crtc_mode_set(struct drm_crtc *crtc,
         if (is_mipi)
                 goto oaktrail_crtc_mode_set_exit;
 
-        refclk = dev_priv->core_freq * 1000;
 
         dpll = 0;               /*BIT16 = 0 for 100MHz reference */
 
-        ok = mrstFindBestPLL(crtc, adjusted_mode->clock, refclk, &clock);
+        refclk = is_sdvo ? 96000 : dev_priv->core_freq * 1000;
+        limit = mrst_limit(crtc, refclk);
+        ok = limit->find_pll(limit, crtc, adjusted_mode->clock,
+                             refclk, &clock);
 
-        if (!ok) {
-                dev_dbg(dev->dev, "mrstFindBestPLL fail in oaktrail_crtc_mode_set.\n");
-        } else {
-                dev_dbg(dev->dev, "oaktrail_crtc_mode_set pixel clock = %d,"
-                         "m = %x, p1 = %x.\n", clock.dot, clock.m,
-                         clock.p1);
+        if (is_sdvo) {
+                /* Convert calculated values to register values */
+                clock.p1 = (1L << (clock.p1 - 1));
+                clock.m -= 2;
+                clock.n = (1L << (clock.n - 1));
         }
 
-        fp = oaktrail_m_converts[(clock.m - MRST_M_MIN)] << 8;
+        if (!ok)
+                DRM_ERROR("Failed to find proper PLL settings");
+
+        mrst_print_pll(&clock);
+
+        if (is_sdvo)
+                fp = clock.n << 16 | clock.m;
+        else
+                fp = oaktrail_m_converts[(clock.m - MRST_M_MIN)] << 8;
 
         dpll |= DPLL_VGA_MODE_DIS;
 
@@ -456,38 +550,43 @@ static int oaktrail_crtc_mode_set(struct drm_crtc *crtc,
 
 
         /* compute bitmask from p1 value */
-        dpll |= (1 << (clock.p1 - 2)) << 17;
+        if (is_sdvo)
+                dpll |= clock.p1 << 16; // dpll |= (1 << (clock.p1 - 1)) << 16;
+        else
+                dpll |= (1 << (clock.p1 - 2)) << 17;
 
         dpll |= DPLL_VCO_ENABLE;
 
-        mrstPrintPll("chosen", &clock);
-
         if (dpll & DPLL_VCO_ENABLE) {
-                REG_WRITE(map->fp0, fp);
-                REG_WRITE(map->dpll, dpll & ~DPLL_VCO_ENABLE);
-                REG_READ(map->dpll);
-                /* Check the DPLLA lock bit PIPEACONF[29] */
-                udelay(150);
+                for (i = 0; i <= need_aux; i++) {
+                        REG_WRITE_WITH_AUX(map->fp0, fp, i);
+                        REG_WRITE_WITH_AUX(map->dpll, dpll & ~DPLL_VCO_ENABLE, i);
+                        REG_READ_WITH_AUX(map->dpll, i);
+                        /* Check the DPLLA lock bit PIPEACONF[29] */
+                        udelay(150);
+                }
         }
 
-        REG_WRITE(map->fp0, fp);
-        REG_WRITE(map->dpll, dpll);
-        REG_READ(map->dpll);
-        /* Wait for the clocks to stabilize. */
-        udelay(150);
+        for (i = 0; i <= need_aux; i++) {
+                REG_WRITE_WITH_AUX(map->fp0, fp, i);
+                REG_WRITE_WITH_AUX(map->dpll, dpll, i);
+                REG_READ_WITH_AUX(map->dpll, i);
+                /* Wait for the clocks to stabilize. */
+                udelay(150);
 
-        /* write it again -- the BIOS does, after all */
-        REG_WRITE(map->dpll, dpll);
-        REG_READ(map->dpll);
-        /* Wait for the clocks to stabilize. */
-        udelay(150);
+                /* write it again -- the BIOS does, after all */
+                REG_WRITE_WITH_AUX(map->dpll, dpll, i);
+                REG_READ_WITH_AUX(map->dpll, i);
+                /* Wait for the clocks to stabilize. */
+                udelay(150);
 
-        REG_WRITE(map->conf, pipeconf);
-        REG_READ(map->conf);
-        gma_wait_for_vblank(dev);
+                REG_WRITE_WITH_AUX(map->conf, pipeconf, i);
+                REG_READ_WITH_AUX(map->conf, i);
+                gma_wait_for_vblank(dev);
 
-        REG_WRITE(map->cntr, dspcntr);
-        gma_wait_for_vblank(dev);
+                REG_WRITE_WITH_AUX(map->cntr, dspcntr, i);
+                gma_wait_for_vblank(dev);
+        }
 
 oaktrail_crtc_mode_set_exit:
         gma_power_end(dev);
@@ -565,3 +664,9 @@ const struct drm_crtc_helper_funcs oaktrail_helper_funcs = {
         .commit = gma_crtc_commit,
 };
 
+/* Not used yet */
+const struct gma_clock_funcs mrst_clock_funcs = {
+        .clock = mrst_lvds_clock,
+        .limit = mrst_limit,
+        .pll_is_valid = gma_pll_is_valid,
+};
diff --git a/drivers/gpu/drm/gma500/oaktrail_device.c b/drivers/gpu/drm/gma500/oaktrail_device.c
index 08747fd7105c..27d3875d895b 100644
--- a/drivers/gpu/drm/gma500/oaktrail_device.c
+++ b/drivers/gpu/drm/gma500/oaktrail_device.c
@@ -40,6 +40,9 @@ static int oaktrail_output_init(struct drm_device *dev)
                 dev_err(dev->dev, "DSI is not supported\n");
         if (dev_priv->hdmi_priv)
                 oaktrail_hdmi_init(dev, &dev_priv->mode_dev);
+
+        psb_intel_sdvo_init(dev, SDVOB);
+
         return 0;
 }
 
@@ -526,6 +529,7 @@ static int oaktrail_chip_setup(struct drm_device *dev)
                 psb_intel_opregion_init(dev);
                 psb_intel_init_bios(dev);
         }
+        gma_intel_setup_gmbus(dev);
         oaktrail_hdmi_setup(dev);
         return 0;
 }
@@ -534,6 +538,7 @@ static void oaktrail_teardown(struct drm_device *dev)
 {
         struct drm_psb_private *dev_priv = dev->dev_private;
 
+        gma_intel_teardown_gmbus(dev);
         oaktrail_hdmi_teardown(dev);
         if (!dev_priv->has_gct)
                 psb_intel_destroy_bios(dev);
@@ -546,6 +551,7 @@ const struct psb_ops oaktrail_chip_ops = {
         .crtcs = 2,
         .hdmi_mask = (1 << 1),
         .lvds_mask = (1 << 0),
+        .sdvo_mask = (1 << 1),
         .cursor_needs_phys = 0,
         .sgx_offset = MRST_SGX_OFFSET,
 
diff --git a/drivers/gpu/drm/gma500/oaktrail_lvds.c b/drivers/gpu/drm/gma500/oaktrail_lvds.c
index e77d7214fca4..4c17c93d8d10 100644
--- a/drivers/gpu/drm/gma500/oaktrail_lvds.c
+++ b/drivers/gpu/drm/gma500/oaktrail_lvds.c
@@ -218,30 +218,6 @@ static const struct drm_encoder_helper_funcs oaktrail_lvds_helper_funcs = {
         .commit = oaktrail_lvds_commit,
 };
 
-static struct drm_display_mode lvds_configuration_modes[] = {
-        /* hard coded fixed mode for TPO LTPS LPJ040K001A */
-        { DRM_MODE("800x480",  DRM_MODE_TYPE_DRIVER, 33264, 800, 836,
-                   846, 1056, 0, 480, 489, 491, 525, 0, 0) },
-        /* hard coded fixed mode for LVDS 800x480 */
-        { DRM_MODE("800x480",  DRM_MODE_TYPE_DRIVER, 30994, 800, 801,
-                   802, 1024, 0, 480, 481, 482, 525, 0, 0) },
-        /* hard coded fixed mode for Samsung 480wsvga LVDS 1024x600@75 */
-        { DRM_MODE("1024x600", DRM_MODE_TYPE_DRIVER, 53990, 1024, 1072,
-                   1104, 1184, 0, 600, 603, 604, 608, 0, 0) },
-        /* hard coded fixed mode for Samsung 480wsvga LVDS 1024x600@75 */
-        { DRM_MODE("1024x600", DRM_MODE_TYPE_DRIVER, 53990, 1024, 1104,
-                   1136, 1184, 0, 600, 603, 604, 608, 0, 0) },
-        /* hard coded fixed mode for Sharp wsvga LVDS 1024x600 */
-        { DRM_MODE("1024x600", DRM_MODE_TYPE_DRIVER, 48885, 1024, 1124,
-                   1204, 1312, 0, 600, 607, 610, 621, 0, 0) },
-        /* hard coded fixed mode for LVDS 1024x768 */
-        { DRM_MODE("1024x768", DRM_MODE_TYPE_DRIVER, 65000, 1024, 1048,
-                   1184, 1344, 0, 768, 771, 777, 806, 0, 0) },
-        /* hard coded fixed mode for LVDS 1366x768 */
-        { DRM_MODE("1366x768", DRM_MODE_TYPE_DRIVER, 77500, 1366, 1430,
-                   1558, 1664, 0, 768, 769, 770, 776, 0, 0) },
-};
-
 /* Returns the panel fixed mode from configuration. */
 
 static void oaktrail_lvds_get_configuration_mode(struct drm_device *dev,
@@ -303,10 +279,10 @@ static void oaktrail_lvds_get_configuration_mode(struct drm_device *dev,
                         mode_dev->panel_fixed_mode =
                                 drm_mode_duplicate(dev,
                                         dev_priv->lfp_lvds_vbt_mode);
-        /* Then guess */
+
+        /* If we still got no mode then bail */
         if (mode_dev->panel_fixed_mode == NULL)
-                mode_dev->panel_fixed_mode
-                        = drm_mode_duplicate(dev, &lvds_configuration_modes[2]);
+                return;
 
         drm_mode_set_name(mode_dev->panel_fixed_mode);
         drm_mode_set_crtcinfo(mode_dev->panel_fixed_mode, 0);
diff --git a/drivers/gpu/drm/gma500/psb_device.c b/drivers/gpu/drm/gma500/psb_device.c
index 697678619bd1..23fb33f1471b 100644
--- a/drivers/gpu/drm/gma500/psb_device.c
+++ b/drivers/gpu/drm/gma500/psb_device.c
@@ -373,6 +373,7 @@ const struct psb_ops psb_chip_ops = {
         .crtcs = 2,
         .hdmi_mask = (1 << 0),
         .lvds_mask = (1 << 1),
+        .sdvo_mask = (1 << 0),
         .cursor_needs_phys = 1,
         .sgx_offset = PSB_SGX_OFFSET,
         .chip_setup = psb_chip_setup,
diff --git a/drivers/gpu/drm/gma500/psb_drv.c b/drivers/gpu/drm/gma500/psb_drv.c
index dd607f820a26..1199180667c9 100644
--- a/drivers/gpu/drm/gma500/psb_drv.c
+++ b/drivers/gpu/drm/gma500/psb_drv.c
@@ -251,6 +251,12 @@ static int psb_driver_unload(struct drm_device *dev)
                         iounmap(dev_priv->sgx_reg);
                         dev_priv->sgx_reg = NULL;
                 }
+                if (dev_priv->aux_reg) {
+                        iounmap(dev_priv->aux_reg);
+                        dev_priv->aux_reg = NULL;
+                }
+                if (dev_priv->aux_pdev)
+                        pci_dev_put(dev_priv->aux_pdev);
 
                 /* Destroy VBT data */
                 psb_intel_destroy_bios(dev);
@@ -266,7 +272,7 @@ static int psb_driver_unload(struct drm_device *dev)
 static int psb_driver_load(struct drm_device *dev, unsigned long chipset)
 {
         struct drm_psb_private *dev_priv;
-        unsigned long resource_start;
+        unsigned long resource_start, resource_len;
         unsigned long irqflags;
         int ret = -ENOMEM;
         struct drm_connector *connector;
@@ -296,6 +302,30 @@ static int psb_driver_load(struct drm_device *dev, unsigned long chipset)
         if (!dev_priv->sgx_reg)
                 goto out_err;
 
+        if (IS_MRST(dev)) {
+                dev_priv->aux_pdev = pci_get_bus_and_slot(0, PCI_DEVFN(3, 0));
+
+                if (dev_priv->aux_pdev) {
+                        resource_start = pci_resource_start(dev_priv->aux_pdev,
+                                                            PSB_AUX_RESOURCE);
+                        resource_len = pci_resource_len(dev_priv->aux_pdev,
+                                                        PSB_AUX_RESOURCE);
+                        dev_priv->aux_reg = ioremap_nocache(resource_start,
+                                                            resource_len);
+                        if (!dev_priv->aux_reg)
+                                goto out_err;
+
+                        DRM_DEBUG_KMS("Found aux vdc");
+                } else {
+                        /* Couldn't find the aux vdc so map to primary vdc */
+                        dev_priv->aux_reg = dev_priv->vdc_reg;
+                        DRM_DEBUG_KMS("Couldn't find aux pci device");
+                }
+                dev_priv->gmbus_reg = dev_priv->aux_reg;
+        } else {
+                dev_priv->gmbus_reg = dev_priv->vdc_reg;
+        }
+
         psb_intel_opregion_setup(dev);
 
         ret = dev_priv->ops->chip_setup(dev);
@@ -449,7 +479,7 @@ static int psb_gamma_ioctl(struct drm_device *dev, void *data,
         obj = drm_mode_object_find(dev, obj_id, DRM_MODE_OBJECT_CONNECTOR);
         if (!obj) {
                 dev_dbg(dev->dev, "Invalid Connector object.\n");
-                return -EINVAL;
+                return -ENOENT;
         }
 
         connector = obj_to_connector(obj);
@@ -491,7 +521,7 @@ static int psb_mode_operation_ioctl(struct drm_device *dev, void *data,
                 obj = drm_mode_object_find(dev, obj_id,
                                         DRM_MODE_OBJECT_CONNECTOR);
                 if (!obj) {
-                        ret = -EINVAL;
+                        ret = -ENOENT;
                         goto mode_op_out;
                 }
 
diff --git a/drivers/gpu/drm/gma500/psb_drv.h b/drivers/gpu/drm/gma500/psb_drv.h
index 0bab46bd73d2..b59e6588c343 100644
--- a/drivers/gpu/drm/gma500/psb_drv.h
+++ b/drivers/gpu/drm/gma500/psb_drv.h
@@ -45,7 +45,7 @@ enum {
 };
 
 #define IS_PSB(dev) (((dev)->pdev->device & 0xfffe) == 0x8108)
-#define IS_MRST(dev) (((dev)->pdev->device & 0xfffc) == 0x4100)
+#define IS_MRST(dev) (((dev)->pdev->device & 0xfff0) == 0x4100)
 #define IS_MFLD(dev) (((dev)->pdev->device & 0xfff8) == 0x0130)
 #define IS_CDV(dev) (((dev)->pdev->device & 0xfff0) == 0x0be0)
 
@@ -75,6 +75,7 @@ enum {
  *      PCI resource identifiers
  */
 #define PSB_MMIO_RESOURCE        0
+#define PSB_AUX_RESOURCE         0
 #define PSB_GATT_RESOURCE        2
 #define PSB_GTT_RESOURCE         3
 /*
@@ -455,6 +456,7 @@ struct psb_ops;
 
 struct drm_psb_private {
         struct drm_device *dev;
+        struct pci_dev *aux_pdev; /* Currently only used by mrst */
         const struct psb_ops *ops;
         const struct psb_offset *regmap;
         
@@ -486,6 +488,7 @@ struct drm_psb_private {
 
         uint8_t __iomem *sgx_reg;
         uint8_t __iomem *vdc_reg;
+        uint8_t __iomem *aux_reg; /* Auxillary vdc pipe regs */
         uint32_t gatt_free_offset;
 
         /*
@@ -532,6 +535,7 @@ struct drm_psb_private {
 
         /* gmbus */
         struct intel_gmbus *gmbus;
+        uint8_t __iomem *gmbus_reg;
 
         /* Used by SDVO */
         int crt_ddc_pin;
@@ -672,6 +676,7 @@ struct psb_ops {
         int sgx_offset;         /* Base offset of SGX device */
         int hdmi_mask;          /* Mask of HDMI CRTCs */
         int lvds_mask;          /* Mask of LVDS CRTCs */
+        int sdvo_mask;          /* Mask of SDVO CRTCs */
         int cursor_needs_phys;  /* If cursor base reg need physical address */
 
         /* Sub functions */
@@ -927,16 +932,58 @@ static inline uint32_t REGISTER_READ(struct drm_device *dev, uint32_t reg)
         return ioread32(dev_priv->vdc_reg + reg);
 }
 
+static inline uint32_t REGISTER_READ_AUX(struct drm_device *dev, uint32_t reg)
+{
+        struct drm_psb_private *dev_priv = dev->dev_private;
+        return ioread32(dev_priv->aux_reg + reg);
+}
+
 #define REG_READ(reg)          REGISTER_READ(dev, (reg))
+#define REG_READ_AUX(reg)      REGISTER_READ_AUX(dev, (reg))
+
+/* Useful for post reads */
+static inline uint32_t REGISTER_READ_WITH_AUX(struct drm_device *dev,
+                                              uint32_t reg, int aux)
+{
+        uint32_t val;
+
+        if (aux)
+                val = REG_READ_AUX(reg);
+        else
+                val = REG_READ(reg);
+
+        return val;
+}
+
+#define REG_READ_WITH_AUX(reg, aux) REGISTER_READ_WITH_AUX(dev, (reg), (aux))
 
 static inline void REGISTER_WRITE(struct drm_device *dev, uint32_t reg,
-                                      uint32_t val)
+                                  uint32_t val)
 {
         struct drm_psb_private *dev_priv = dev->dev_private;
         iowrite32((val), dev_priv->vdc_reg + (reg));
 }
 
+static inline void REGISTER_WRITE_AUX(struct drm_device *dev, uint32_t reg,
+                                      uint32_t val)
+{
+        struct drm_psb_private *dev_priv = dev->dev_private;
+        iowrite32((val), dev_priv->aux_reg + (reg));
+}
+
 #define REG_WRITE(reg, val)     REGISTER_WRITE(dev, (reg), (val))
+#define REG_WRITE_AUX(reg, val) REGISTER_WRITE_AUX(dev, (reg), (val))
+
+static inline void REGISTER_WRITE_WITH_AUX(struct drm_device *dev, uint32_t reg,
+                                      uint32_t val, int aux)
+{
+        if (aux)
+                REG_WRITE_AUX(reg, val);
+        else
+                REG_WRITE(reg, val);
+}
+
+#define REG_WRITE_WITH_AUX(reg, val, aux) REGISTER_WRITE_WITH_AUX(dev, (reg), (val), (aux))
 
 static inline void REGISTER_WRITE16(struct drm_device *dev,
                                         uint32_t reg, uint32_t val)
diff --git a/drivers/gpu/drm/gma500/psb_intel_display.c b/drivers/gpu/drm/gma500/psb_intel_display.c
index 97f8a03fee43..c8841ac6c8f1 100644
--- a/drivers/gpu/drm/gma500/psb_intel_display.c
+++ b/drivers/gpu/drm/gma500/psb_intel_display.c
@@ -572,7 +572,7 @@ int psb_intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
 
         if (!drmmode_obj) {
                 dev_err(dev->dev, "no such CRTC id\n");
-                return -EINVAL;
+                return -ENOENT;
         }
 
         crtc = to_gma_crtc(obj_to_crtc(drmmode_obj));
diff --git a/drivers/gpu/drm/gma500/psb_intel_sdvo.c b/drivers/gpu/drm/gma500/psb_intel_sdvo.c
index 6f01cdf5e125..07d3a9e6d79b 100644
--- a/drivers/gpu/drm/gma500/psb_intel_sdvo.c
+++ b/drivers/gpu/drm/gma500/psb_intel_sdvo.c
@@ -228,24 +228,26 @@ static void psb_intel_sdvo_write_sdvox(struct psb_intel_sdvo *psb_intel_sdvo, u3
 {
         struct drm_device *dev = psb_intel_sdvo->base.base.dev;
         u32 bval = val, cval = val;
-        int i;
+        int i, j;
+        int need_aux = IS_MRST(dev) ? 1 : 0;
 
-        if (psb_intel_sdvo->sdvo_reg == SDVOB) {
-                cval = REG_READ(SDVOC);
-        } else {
-                bval = REG_READ(SDVOB);
-        }
-        /*
-         * Write the registers twice for luck. Sometimes,
-         * writing them only once doesn't appear to 'stick'.
-         * The BIOS does this too. Yay, magic
-         */
-        for (i = 0; i < 2; i++)
-        {
-                REG_WRITE(SDVOB, bval);
-                REG_READ(SDVOB);
-                REG_WRITE(SDVOC, cval);
-                REG_READ(SDVOC);
+        for (j = 0; j <= need_aux; j++) {
+                if (psb_intel_sdvo->sdvo_reg == SDVOB)
+                        cval = REG_READ_WITH_AUX(SDVOC, j);
+                else
+                        bval = REG_READ_WITH_AUX(SDVOB, j);
+
+                /*
+                * Write the registers twice for luck. Sometimes,
+                * writing them only once doesn't appear to 'stick'.
+                * The BIOS does this too. Yay, magic
+                */
+                for (i = 0; i < 2; i++) {
+                        REG_WRITE_WITH_AUX(SDVOB, bval, j);
+                        REG_READ_WITH_AUX(SDVOB, j);
+                        REG_WRITE_WITH_AUX(SDVOC, cval, j);
+                        REG_READ_WITH_AUX(SDVOC, j);
+                }
         }
 }
 
@@ -995,6 +997,7 @@ static void psb_intel_sdvo_mode_set(struct drm_encoder *encoder,
         struct psb_intel_sdvo_dtd input_dtd;
         int pixel_multiplier = psb_intel_mode_get_pixel_multiplier(adjusted_mode);
         int rate;
+        int need_aux = IS_MRST(dev) ? 1 : 0;
 
         if (!mode)
                 return;
@@ -1060,7 +1063,11 @@ static void psb_intel_sdvo_mode_set(struct drm_encoder *encoder,
                 return;
 
         /* Set the SDVO control regs. */
-        sdvox = REG_READ(psb_intel_sdvo->sdvo_reg);
+        if (need_aux)
+                sdvox = REG_READ_AUX(psb_intel_sdvo->sdvo_reg);
+        else
+                sdvox = REG_READ(psb_intel_sdvo->sdvo_reg);
+
         switch (psb_intel_sdvo->sdvo_reg) {
         case SDVOB:
                 sdvox &= SDVOB_PRESERVE_MASK;
@@ -1090,6 +1097,8 @@ static void psb_intel_sdvo_dpms(struct drm_encoder *encoder, int mode)
         struct drm_device *dev = encoder->dev;
         struct psb_intel_sdvo *psb_intel_sdvo = to_psb_intel_sdvo(encoder);
         u32 temp;
+        int i;
+        int need_aux = IS_MRST(dev) ? 1 : 0;
 
         switch (mode) {
         case DRM_MODE_DPMS_ON:
@@ -1108,19 +1117,27 @@ static void psb_intel_sdvo_dpms(struct drm_encoder *encoder, int mode)
                         psb_intel_sdvo_set_encoder_power_state(psb_intel_sdvo, mode);
 
                 if (mode == DRM_MODE_DPMS_OFF) {
-                        temp = REG_READ(psb_intel_sdvo->sdvo_reg);
+                        if (need_aux)
+                                temp = REG_READ_AUX(psb_intel_sdvo->sdvo_reg);
+                        else
+                                temp = REG_READ(psb_intel_sdvo->sdvo_reg);
+
                         if ((temp & SDVO_ENABLE) != 0) {
                                 psb_intel_sdvo_write_sdvox(psb_intel_sdvo, temp & ~SDVO_ENABLE);
                         }
                 }
         } else {
                 bool input1, input2;
-                int i;
                 u8 status;
 
-                temp = REG_READ(psb_intel_sdvo->sdvo_reg);
+                if (need_aux)
+                        temp = REG_READ_AUX(psb_intel_sdvo->sdvo_reg);
+                else
+                        temp = REG_READ(psb_intel_sdvo->sdvo_reg);
+
                 if ((temp & SDVO_ENABLE) == 0)
                         psb_intel_sdvo_write_sdvox(psb_intel_sdvo, temp | SDVO_ENABLE);
+
                 for (i = 0; i < 2; i++)
                         gma_wait_for_vblank(dev);
 
diff --git a/drivers/gpu/drm/i2c/tda998x_drv.c b/drivers/gpu/drm/i2c/tda998x_drv.c
index 60e84043aa34..400b0c4a10fb 100644
--- a/drivers/gpu/drm/i2c/tda998x_drv.c
+++ b/drivers/gpu/drm/i2c/tda998x_drv.c
@@ -17,6 +17,7 @@
 
 
 
+#include <linux/hdmi.h>
 #include <linux/module.h>
 
 #include <drm/drmP.h>
@@ -549,6 +550,8 @@ tda998x_write_avi(struct drm_encoder *encoder, struct drm_display_mode *mode)
         buf[HB(0)] = 0x82;
         buf[HB(1)] = 0x02;
         buf[HB(2)] = 13;
+        buf[PB(1)] = HDMI_SCAN_MODE_UNDERSCAN;
+        buf[PB(3)] = HDMI_QUANTIZATION_RANGE_FULL << 2;
         buf[PB(4)] = drm_match_cea_mode(mode);
 
         tda998x_write_if(encoder, DIP_IF_FLAGS_IF2, REG_IF2_HB0, buf,
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 506f8efeb519..d1491f8e0f39 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -586,7 +586,53 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
         if (ret)
                 return ret;
 
-        if (IS_VALLEYVIEW(dev)) {
+        if (INTEL_INFO(dev)->gen >= 8) {
+                int i;
+                seq_printf(m, "Master Interrupt Control:\t%08x\n",
+                           I915_READ(GEN8_MASTER_IRQ));
+
+                for (i = 0; i < 4; i++) {
+                        seq_printf(m, "GT Interrupt IMR %d:\t%08x\n",
+                                   i, I915_READ(GEN8_GT_IMR(i)));
+                        seq_printf(m, "GT Interrupt IIR %d:\t%08x\n",
+                                   i, I915_READ(GEN8_GT_IIR(i)));
+                        seq_printf(m, "GT Interrupt IER %d:\t%08x\n",
+                                   i, I915_READ(GEN8_GT_IER(i)));
+                }
+
+                for_each_pipe(i) {
+                        seq_printf(m, "Pipe %c IMR:\t%08x\n",
+                                   pipe_name(i),
+                                   I915_READ(GEN8_DE_PIPE_IMR(i)));
+                        seq_printf(m, "Pipe %c IIR:\t%08x\n",
+                                   pipe_name(i),
+                                   I915_READ(GEN8_DE_PIPE_IIR(i)));
+                        seq_printf(m, "Pipe %c IER:\t%08x\n",
+                                   pipe_name(i),
+                                   I915_READ(GEN8_DE_PIPE_IER(i)));
+                }
+
+                seq_printf(m, "Display Engine port interrupt mask:\t%08x\n",
+                           I915_READ(GEN8_DE_PORT_IMR));
+                seq_printf(m, "Display Engine port interrupt identity:\t%08x\n",
+                           I915_READ(GEN8_DE_PORT_IIR));
+                seq_printf(m, "Display Engine port interrupt enable:\t%08x\n",
+                           I915_READ(GEN8_DE_PORT_IER));
+
+                seq_printf(m, "Display Engine misc interrupt mask:\t%08x\n",
+                           I915_READ(GEN8_DE_MISC_IMR));
+                seq_printf(m, "Display Engine misc interrupt identity:\t%08x\n",
+                           I915_READ(GEN8_DE_MISC_IIR));
+                seq_printf(m, "Display Engine misc interrupt enable:\t%08x\n",
+                           I915_READ(GEN8_DE_MISC_IER));
+
+                seq_printf(m, "PCU interrupt mask:\t%08x\n",
+                           I915_READ(GEN8_PCU_IMR));
+                seq_printf(m, "PCU interrupt identity:\t%08x\n",
+                           I915_READ(GEN8_PCU_IIR));
+                seq_printf(m, "PCU interrupt enable:\t%08x\n",
+                           I915_READ(GEN8_PCU_IER));
+        } else if (IS_VALLEYVIEW(dev)) {
                 seq_printf(m, "Display IER:\t%08x\n",
                            I915_READ(VLV_IER));
                 seq_printf(m, "Display IIR:\t%08x\n",
@@ -658,7 +704,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
         seq_printf(m, "Interrupts received: %d\n",
                    atomic_read(&dev_priv->irq_received));
         for_each_ring(ring, dev_priv, i) {
-                if (IS_GEN6(dev) || IS_GEN7(dev)) {
+                if (INTEL_INFO(dev)->gen >= 6) {
                         seq_printf(m,
                                    "Graphics Interrupt mask (%s):       %08x\n",
                                    ring->name, I915_READ_IMR(ring));
@@ -1576,7 +1622,7 @@ static int i915_swizzle_info(struct seq_file *m, void *data)
                            I915_READ16(C0DRB3));
                 seq_printf(m, "C1DRB3 = 0x%04x\n",
                            I915_READ16(C1DRB3));
-        } else if (IS_GEN6(dev) || IS_GEN7(dev)) {
+        } else if (INTEL_INFO(dev)->gen >= 6) {
                 seq_printf(m, "MAD_DIMM_C0 = 0x%08x\n",
                            I915_READ(MAD_DIMM_C0));
                 seq_printf(m, "MAD_DIMM_C1 = 0x%08x\n",
@@ -1585,8 +1631,12 @@ static int i915_swizzle_info(struct seq_file *m, void *data)
                            I915_READ(MAD_DIMM_C2));
                 seq_printf(m, "TILECTL = 0x%08x\n",
                            I915_READ(TILECTL));
-                seq_printf(m, "ARB_MODE = 0x%08x\n",
-                           I915_READ(ARB_MODE));
+                if (IS_GEN8(dev))
+                        seq_printf(m, "GAMTARBMODE = 0x%08x\n",
+                                   I915_READ(GAMTARBMODE));
+                else
+                        seq_printf(m, "ARB_MODE = 0x%08x\n",
+                                   I915_READ(ARB_MODE));
                 seq_printf(m, "DISP_ARB_CTL = 0x%08x\n",
                            I915_READ(DISP_ARB_CTL));
         }
@@ -1595,18 +1645,37 @@ static int i915_swizzle_info(struct seq_file *m, void *data)
         return 0;
 }
 
-static int i915_ppgtt_info(struct seq_file *m, void *data)
+static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)
 {
-        struct drm_info_node *node = (struct drm_info_node *) m->private;
-        struct drm_device *dev = node->minor->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring;
-        int i, ret;
+        struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
+        int unused, i;
 
+        if (!ppgtt)
+                return;
+
+        seq_printf(m, "Page directories: %d\n", ppgtt->num_pd_pages);
+        seq_printf(m, "Page tables: %d\n", ppgtt->num_pt_pages);
+        for_each_ring(ring, dev_priv, unused) {
+                seq_printf(m, "%s\n", ring->name);
+                for (i = 0; i < 4; i++) {
+                        u32 offset = 0x270 + i * 8;
+                        u64 pdp = I915_READ(ring->mmio_base + offset + 4);
+                        pdp <<= 32;
+                        pdp |= I915_READ(ring->mmio_base + offset);
+                        for (i = 0; i < 4; i++)
+                                seq_printf(m, "\tPDP%d 0x%016llx\n", i, pdp);
+                }
+        }
+}
+
+static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring;
+        int i;
 
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
         if (INTEL_INFO(dev)->gen == 6)
                 seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(GFX_MODE));
 
@@ -1625,6 +1694,22 @@ static int i915_ppgtt_info(struct seq_file *m, void *data)
                 seq_printf(m, "pd gtt offset: 0x%08x\n", ppgtt->pd_offset);
         }
         seq_printf(m, "ECOCHK: 0x%08x\n", I915_READ(GAM_ECOCHK));
+}
+
+static int i915_ppgtt_info(struct seq_file *m, void *data)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+
+        int ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
+
+        if (INTEL_INFO(dev)->gen >= 8)
+                gen8_ppgtt_info(m, dev);
+        else if (INTEL_INFO(dev)->gen >= 6)
+                gen6_ppgtt_info(m, dev);
+
         mutex_unlock(&dev->struct_mutex);
 
         return 0;
@@ -2955,7 +3040,7 @@ static int i915_debugfs_create(struct dentry *root,
         return drm_add_fake_info_node(minor, ent, fops);
 }
 
-static struct drm_info_list i915_debugfs_list[] = {
+static const struct drm_info_list i915_debugfs_list[] = {
         {"i915_capabilities", i915_capabilities, 0},
         {"i915_gem_objects", i915_gem_object_info, 0},
         {"i915_gem_gtt", i915_gem_gtt_info, 0},
@@ -2997,7 +3082,7 @@ static struct drm_info_list i915_debugfs_list[] = {
 };
 #define I915_DEBUGFS_ENTRIES ARRAY_SIZE(i915_debugfs_list)
 
-static struct i915_debugfs_files {
+static const struct i915_debugfs_files {
         const char *name;
         const struct file_operations *fops;
 } i915_debugfs_files[] = {
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index 00d74f816a72..25acbb5eca6e 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -1486,7 +1486,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 
         spin_lock_init(&dev_priv->irq_lock);
         spin_lock_init(&dev_priv->gpu_error.lock);
-        spin_lock_init(&dev_priv->backlight.lock);
+        spin_lock_init(&dev_priv->backlight_lock);
         spin_lock_init(&dev_priv->uncore.lock);
         spin_lock_init(&dev_priv->mm.object_stat_lock);
         mutex_init(&dev_priv->dpio_lock);
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 92ad319164d7..c2e00ed23195 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -335,6 +335,24 @@ static const struct intel_device_info intel_haswell_m_info = {
         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
 };
 
+static const struct intel_device_info intel_broadwell_d_info = {
+        .is_preliminary = 1,
+        .gen = 8, .num_pipes = 3,
+        .need_gfx_hws = 1, .has_hotplug = 1,
+        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
+        .has_llc = 1,
+        .has_ddi = 1,
+};
+
+static const struct intel_device_info intel_broadwell_m_info = {
+        .is_preliminary = 1,
+        .gen = 8, .is_mobile = 1, .num_pipes = 3,
+        .need_gfx_hws = 1, .has_hotplug = 1,
+        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
+        .has_llc = 1,
+        .has_ddi = 1,
+};
+
 /*
  * Make sure any device matches here are from most specific to most
  * general.  For example, since the Quanta match is based on the subsystem
@@ -366,7 +384,9 @@ static const struct intel_device_info intel_haswell_m_info = {
         INTEL_HSW_D_IDS(&intel_haswell_d_info), \
         INTEL_HSW_M_IDS(&intel_haswell_m_info), \
         INTEL_VLV_M_IDS(&intel_valleyview_m_info),      \
-        INTEL_VLV_D_IDS(&intel_valleyview_d_info)
+        INTEL_VLV_D_IDS(&intel_valleyview_d_info),      \
+        INTEL_BDW_M_IDS(&intel_broadwell_m_info),       \
+        INTEL_BDW_D_IDS(&intel_broadwell_d_info)
 
 static const struct pci_device_id pciidlist[] = {               /* aka */
         INTEL_PCI_IDS,
@@ -427,6 +447,12 @@ void intel_detect_pch(struct drm_device *dev)
                                 DRM_DEBUG_KMS("Found LynxPoint PCH\n");
                                 WARN_ON(!IS_HASWELL(dev));
                                 WARN_ON(IS_ULT(dev));
+                        } else if (IS_BROADWELL(dev)) {
+                                dev_priv->pch_type = PCH_LPT;
+                                dev_priv->pch_id =
+                                        INTEL_PCH_LPT_LP_DEVICE_ID_TYPE;
+                                DRM_DEBUG_KMS("This is Broadwell, assuming "
+                                              "LynxPoint LP PCH\n");
                         } else if (id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
                                 dev_priv->pch_type = PCH_LPT;
                                 DRM_DEBUG_KMS("Found LynxPoint LP PCH\n");
@@ -451,6 +477,12 @@ bool i915_semaphore_is_enabled(struct drm_device *dev)
         if (INTEL_INFO(dev)->gen < 6)
                 return 0;
 
+        /* Until we get further testing... */
+        if (IS_GEN8(dev)) {
+                WARN_ON(!i915_preliminary_hw_support);
+                return 0;
+        }
+
         if (i915_semaphores >= 0)
                 return i915_semaphores;
 
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 6d2a9a1c8379..6b96e91c6f1a 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -54,6 +54,7 @@
 #define DRIVER_DATE             "20080730"
 
 enum pipe {
+        INVALID_PIPE = -1,
         PIPE_A = 0,
         PIPE_B,
         PIPE_C,
@@ -129,6 +130,10 @@ enum intel_display_power_domain {
 #define HSW_ALWAYS_ON_POWER_DOMAINS (           \
         BIT(POWER_DOMAIN_PIPE_A) |              \
         BIT(POWER_DOMAIN_TRANSCODER_EDP))
+#define BDW_ALWAYS_ON_POWER_DOMAINS (           \
+        BIT(POWER_DOMAIN_PIPE_A) |              \
+        BIT(POWER_DOMAIN_TRANSCODER_EDP) |      \
+        BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER))
 
 enum hpd_pin {
         HPD_NONE = 0,
@@ -254,6 +259,7 @@ struct intel_opregion {
         struct opregion_asle __iomem *asle;
         void __iomem *vbt;
         u32 __iomem *lid_state;
+        struct work_struct asle_work;
 };
 #define OPREGION_SIZE            (8*1024)
 
@@ -357,6 +363,7 @@ struct drm_i915_error_state {
         enum intel_ring_hangcheck_action hangcheck_action[I915_NUM_RINGS];
 };
 
+struct intel_connector;
 struct intel_crtc_config;
 struct intel_crtc;
 struct intel_limit;
@@ -419,6 +426,13 @@ struct drm_i915_display_funcs {
         /* render clock increase/decrease */
         /* display clock increase/decrease */
         /* pll clock increase/decrease */
+
+        int (*setup_backlight)(struct intel_connector *connector);
+        uint32_t (*get_backlight)(struct intel_connector *connector);
+        void (*set_backlight)(struct intel_connector *connector,
+                              uint32_t level);
+        void (*disable_backlight)(struct intel_connector *connector);
+        void (*enable_backlight)(struct intel_connector *connector);
 };
 
 struct intel_uncore_funcs {
@@ -585,10 +599,21 @@ struct i915_gtt {
 struct i915_hw_ppgtt {
         struct i915_address_space base;
         unsigned num_pd_entries;
-        struct page **pt_pages;
-        uint32_t pd_offset;
-        dma_addr_t *pt_dma_addr;
-
+        union {
+                struct page **pt_pages;
+                struct page *gen8_pt_pages;
+        };
+        struct page *pd_pages;
+        int num_pd_pages;
+        int num_pt_pages;
+        union {
+                uint32_t pd_offset;
+                dma_addr_t pd_dma_addr[4];
+        };
+        union {
+                dma_addr_t *pt_dma_addr;
+                dma_addr_t *gen8_pt_dma_addr[4];
+        };
         int (*enable)(struct drm_device *dev);
 };
 
@@ -703,7 +728,6 @@ enum intel_sbi_destination {
 #define QUIRK_PIPEA_FORCE (1<<0)
 #define QUIRK_LVDS_SSC_DISABLE (1<<1)
 #define QUIRK_INVERT_BRIGHTNESS (1<<2)
-#define QUIRK_NO_PCH_PWM_ENABLE (1<<3)
 
 struct intel_fbdev;
 struct intel_fbc_work;
@@ -755,6 +779,7 @@ struct i915_suspend_saved_registers {
         u32 saveBLC_HIST_CTL;
         u32 saveBLC_PWM_CTL;
         u32 saveBLC_PWM_CTL2;
+        u32 saveBLC_HIST_CTL_B;
         u32 saveBLC_CPU_PWM_CTL;
         u32 saveBLC_CPU_PWM_CTL2;
         u32 saveFPB0;
@@ -1325,7 +1350,10 @@ typedef struct drm_i915_private {
         struct mutex dpio_lock;
 
         /** Cached value of IMR to avoid reads in updating the bitfield */
-        u32 irq_mask;
+        union {
+                u32 irq_mask;
+                u32 de_irq_mask[I915_MAX_PIPES];
+        };
         u32 gt_irq_mask;
         u32 pm_irq_mask;
 
@@ -1353,13 +1381,8 @@ typedef struct drm_i915_private {
         struct intel_overlay *overlay;
         unsigned int sprite_scaling_enabled;
 
-        /* backlight */
-        struct {
-                int level;
-                bool enabled;
-                spinlock_t lock; /* bl registers and the above bl fields */
-                struct backlight_device *device;
-        } backlight;
+        /* backlight registers and fields in struct intel_panel */
+        spinlock_t backlight_lock;
 
         /* LVDS info */
         bool no_aux_handshake;
@@ -1736,11 +1759,17 @@ struct drm_i915_file_private {
                                  (dev)->pdev->device == 0x010A)
 #define IS_VALLEYVIEW(dev)      (INTEL_INFO(dev)->is_valleyview)
 #define IS_HASWELL(dev) (INTEL_INFO(dev)->is_haswell)
+#define IS_BROADWELL(dev)       (INTEL_INFO(dev)->gen == 8)
 #define IS_MOBILE(dev)          (INTEL_INFO(dev)->is_mobile)
 #define IS_HSW_EARLY_SDV(dev)   (IS_HASWELL(dev) && \
                                  ((dev)->pdev->device & 0xFF00) == 0x0C00)
-#define IS_ULT(dev)             (IS_HASWELL(dev) && \
+#define IS_BDW_ULT(dev)         (IS_BROADWELL(dev) && \
+                                 (((dev)->pdev->device & 0xf) == 0x2  || \
+                                 ((dev)->pdev->device & 0xf) == 0x6 || \
+                                 ((dev)->pdev->device & 0xf) == 0xe))
+#define IS_HSW_ULT(dev)         (IS_HASWELL(dev) && \
                                  ((dev)->pdev->device & 0xFF00) == 0x0A00)
+#define IS_ULT(dev)             (IS_HSW_ULT(dev) || IS_BDW_ULT(dev))
 #define IS_HSW_GT3(dev)         (IS_HASWELL(dev) && \
                                  ((dev)->pdev->device & 0x00F0) == 0x0020)
 #define IS_PRELIMINARY_HW(intel_info) ((intel_info)->is_preliminary)
@@ -1757,6 +1786,7 @@ struct drm_i915_file_private {
 #define IS_GEN5(dev)    (INTEL_INFO(dev)->gen == 5)
 #define IS_GEN6(dev)    (INTEL_INFO(dev)->gen == 6)
 #define IS_GEN7(dev)    (INTEL_INFO(dev)->gen == 7)
+#define IS_GEN8(dev)    (INTEL_INFO(dev)->gen == 8)
 
 #define RENDER_RING             (1<<RCS)
 #define BSD_RING                (1<<VCS)
@@ -1793,12 +1823,12 @@ struct drm_i915_file_private {
 #define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr)
 #define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc)
 
-#define HAS_IPS(dev)            (IS_ULT(dev))
+#define HAS_IPS(dev)            (IS_ULT(dev) || IS_BROADWELL(dev))
 
 #define HAS_DDI(dev)            (INTEL_INFO(dev)->has_ddi)
-#define HAS_POWER_WELL(dev)     (IS_HASWELL(dev))
+#define HAS_POWER_WELL(dev)     (IS_HASWELL(dev) || IS_BROADWELL(dev))
 #define HAS_FPGA_DBG_UNCLAIMED(dev)     (INTEL_INFO(dev)->has_fpga_dbg)
-#define HAS_PSR(dev)            (IS_HASWELL(dev))
+#define HAS_PSR(dev)            (IS_HASWELL(dev) || IS_BROADWELL(dev))
 
 #define INTEL_PCH_DEVICE_ID_MASK                0xff00
 #define INTEL_PCH_IBX_DEVICE_ID_TYPE            0x3b00
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index bc528201caca..40d9dcf858bf 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -2954,6 +2954,7 @@ static void i915_gem_write_fence(struct drm_device *dev, int reg,
              obj->stride, obj->tiling_mode);
 
         switch (INTEL_INFO(dev)->gen) {
+        case 8:
         case 7:
         case 6:
         case 5:
@@ -4361,6 +4362,8 @@ void i915_gem_init_swizzling(struct drm_device *dev)
                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
         else if (IS_GEN7(dev))
                 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
+        else if (IS_GEN8(dev))
+                I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
         else
                 BUG();
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index 4625670bcadb..2ec122a63406 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -117,6 +117,9 @@ static int get_context_size(struct drm_device *dev)
                 else
                         ret = GEN7_CXT_TOTAL_SIZE(reg) * 64;
                 break;
+        case 8:
+                ret = GEN8_CXT_TOTAL_SIZE;
+                break;
         default:
                 BUG();
         }
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 0ce0d47e4b0f..885d595e0e02 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -212,6 +212,7 @@ static int
 relocate_entry_cpu(struct drm_i915_gem_object *obj,
                    struct drm_i915_gem_relocation_entry *reloc)
 {
+        struct drm_device *dev = obj->base.dev;
         uint32_t page_offset = offset_in_page(reloc->offset);
         char *vaddr;
         int ret = -EINVAL;
@@ -223,6 +224,19 @@ relocate_entry_cpu(struct drm_i915_gem_object *obj,
         vaddr = kmap_atomic(i915_gem_object_get_page(obj,
                                 reloc->offset >> PAGE_SHIFT));
         *(uint32_t *)(vaddr + page_offset) = reloc->delta;
+
+        if (INTEL_INFO(dev)->gen >= 8) {
+                page_offset = offset_in_page(page_offset + sizeof(uint32_t));
+
+                if (page_offset == 0) {
+                        kunmap_atomic(vaddr);
+                        vaddr = kmap_atomic(i915_gem_object_get_page(obj,
+                            (reloc->offset + sizeof(uint32_t)) >> PAGE_SHIFT));
+                }
+
+                *(uint32_t *)(vaddr + page_offset) = 0;
+        }
+
         kunmap_atomic(vaddr);
 
         return 0;
@@ -253,6 +267,21 @@ relocate_entry_gtt(struct drm_i915_gem_object *obj,
         reloc_entry = (uint32_t __iomem *)
                 (reloc_page + offset_in_page(reloc->offset));
         iowrite32(reloc->delta, reloc_entry);
+
+        if (INTEL_INFO(dev)->gen >= 8) {
+                reloc_entry += 1;
+
+                if (offset_in_page(reloc->offset + sizeof(uint32_t)) == 0) {
+                        io_mapping_unmap_atomic(reloc_page);
+                        reloc_page = io_mapping_map_atomic_wc(
+                                        dev_priv->gtt.mappable,
+                                        reloc->offset + sizeof(uint32_t));
+                        reloc_entry = reloc_page;
+                }
+
+                iowrite32(0, reloc_entry);
+        }
+
         io_mapping_unmap_atomic(reloc_page);
 
         return 0;
@@ -323,7 +352,8 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
                 return 0;
 
         /* Check that the relocation address is valid... */
-        if (unlikely(reloc->offset > obj->base.size - 4)) {
+        if (unlikely(reloc->offset >
+                obj->base.size - (INTEL_INFO(dev)->gen >= 8 ? 8 : 4))) {
                 DRM_DEBUG("Relocation beyond object bounds: "
                           "obj %p target %d offset %d size %d.\n",
                           obj, reloc->target_handle,
@@ -1116,8 +1146,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 
         /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
          * batch" bit. Hence we need to pin secure batches into the global gtt.
-         * hsw should have this fixed, but let's be paranoid and do it
-         * unconditionally for now. */
+         * hsw should have this fixed, but bdw mucks it up again. */
         if (flags & I915_DISPATCH_SECURE && !batch_obj->has_global_gtt_mapping)
                 i915_gem_gtt_bind_object(batch_obj, batch_obj->cache_level);
 
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 2b4c530d74a3..efb5dab61c81 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -30,6 +30,8 @@
 
 #define GEN6_PPGTT_PD_ENTRIES 512
 #define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t))
+typedef uint64_t gen8_gtt_pte_t;
+typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;
 
 /* PPGTT stuff */
 #define GEN6_GTT_ADDR_ENCODE(addr)      ((addr) | (((addr) >> 28) & 0xff0))
@@ -57,6 +59,41 @@
 #define HSW_WB_ELLC_LLC_AGE0            HSW_CACHEABILITY_CONTROL(0xb)
 #define HSW_WT_ELLC_LLC_AGE0            HSW_CACHEABILITY_CONTROL(0x6)
 
+#define GEN8_PTES_PER_PAGE              (PAGE_SIZE / sizeof(gen8_gtt_pte_t))
+#define GEN8_PDES_PER_PAGE              (PAGE_SIZE / sizeof(gen8_ppgtt_pde_t))
+#define GEN8_LEGACY_PDPS                4
+
+#define PPAT_UNCACHED_INDEX             (_PAGE_PWT | _PAGE_PCD)
+#define PPAT_CACHED_PDE_INDEX           0 /* WB LLC */
+#define PPAT_CACHED_INDEX               _PAGE_PAT /* WB LLCeLLC */
+#define PPAT_DISPLAY_ELLC_INDEX         _PAGE_PCD /* WT eLLC */
+
+static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
+                                             enum i915_cache_level level,
+                                             bool valid)
+{
+        gen8_gtt_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
+        pte |= addr;
+        if (level != I915_CACHE_NONE)
+                pte |= PPAT_CACHED_INDEX;
+        else
+                pte |= PPAT_UNCACHED_INDEX;
+        return pte;
+}
+
+static inline gen8_ppgtt_pde_t gen8_pde_encode(struct drm_device *dev,
+                                             dma_addr_t addr,
+                                             enum i915_cache_level level)
+{
+        gen8_ppgtt_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
+        pde |= addr;
+        if (level != I915_CACHE_NONE)
+                pde |= PPAT_CACHED_PDE_INDEX;
+        else
+                pde |= PPAT_UNCACHED_INDEX;
+        return pde;
+}
+
 static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
                                      enum i915_cache_level level,
                                      bool valid)
@@ -158,6 +195,257 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
         return pte;
 }
 
+/* Broadwell Page Directory Pointer Descriptors */
+static int gen8_write_pdp(struct intel_ring_buffer *ring, unsigned entry,
+                           uint64_t val)
+{
+        int ret;
+
+        BUG_ON(entry >= 4);
+
+        ret = intel_ring_begin(ring, 6);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
+        intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
+        intel_ring_emit(ring, (u32)(val >> 32));
+        intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
+        intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
+        intel_ring_emit(ring, (u32)(val));
+        intel_ring_advance(ring);
+
+        return 0;
+}
+
+static int gen8_ppgtt_enable(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring;
+        struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
+        int i, j, ret;
+
+        /* bit of a hack to find the actual last used pd */
+        int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE;
+
+        for_each_ring(ring, dev_priv, j) {
+                I915_WRITE(RING_MODE_GEN7(ring),
+                           _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
+        }
+
+        for (i = used_pd - 1; i >= 0; i--) {
+                dma_addr_t addr = ppgtt->pd_dma_addr[i];
+                for_each_ring(ring, dev_priv, j) {
+                        ret = gen8_write_pdp(ring, i, addr);
+                        if (ret)
+                                return ret;
+                }
+        }
+        return 0;
+}
+
+static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
+                                   unsigned first_entry,
+                                   unsigned num_entries,
+                                   bool use_scratch)
+{
+        struct i915_hw_ppgtt *ppgtt =
+                container_of(vm, struct i915_hw_ppgtt, base);
+        gen8_gtt_pte_t *pt_vaddr, scratch_pte;
+        unsigned act_pt = first_entry / GEN8_PTES_PER_PAGE;
+        unsigned first_pte = first_entry % GEN8_PTES_PER_PAGE;
+        unsigned last_pte, i;
+
+        scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,
+                                      I915_CACHE_LLC, use_scratch);
+
+        while (num_entries) {
+                struct page *page_table = &ppgtt->gen8_pt_pages[act_pt];
+
+                last_pte = first_pte + num_entries;
+                if (last_pte > GEN8_PTES_PER_PAGE)
+                        last_pte = GEN8_PTES_PER_PAGE;
+
+                pt_vaddr = kmap_atomic(page_table);
+
+                for (i = first_pte; i < last_pte; i++)
+                        pt_vaddr[i] = scratch_pte;
+
+                kunmap_atomic(pt_vaddr);
+
+                num_entries -= last_pte - first_pte;
+                first_pte = 0;
+                act_pt++;
+        }
+}
+
+static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
+                                      struct sg_table *pages,
+                                      unsigned first_entry,
+                                      enum i915_cache_level cache_level)
+{
+        struct i915_hw_ppgtt *ppgtt =
+                container_of(vm, struct i915_hw_ppgtt, base);
+        gen8_gtt_pte_t *pt_vaddr;
+        unsigned act_pt = first_entry / GEN8_PTES_PER_PAGE;
+        unsigned act_pte = first_entry % GEN8_PTES_PER_PAGE;
+        struct sg_page_iter sg_iter;
+
+        pt_vaddr = kmap_atomic(&ppgtt->gen8_pt_pages[act_pt]);
+        for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
+                dma_addr_t page_addr;
+
+                page_addr = sg_dma_address(sg_iter.sg) +
+                                (sg_iter.sg_pgoffset << PAGE_SHIFT);
+                pt_vaddr[act_pte] = gen8_pte_encode(page_addr, cache_level,
+                                                    true);
+                if (++act_pte == GEN8_PTES_PER_PAGE) {
+                        kunmap_atomic(pt_vaddr);
+                        act_pt++;
+                        pt_vaddr = kmap_atomic(&ppgtt->gen8_pt_pages[act_pt]);
+                        act_pte = 0;
+
+                }
+        }
+        kunmap_atomic(pt_vaddr);
+}
+
+static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
+{
+        struct i915_hw_ppgtt *ppgtt =
+                container_of(vm, struct i915_hw_ppgtt, base);
+        int i, j;
+
+        for (i = 0; i < ppgtt->num_pd_pages ; i++) {
+                if (ppgtt->pd_dma_addr[i]) {
+                        pci_unmap_page(ppgtt->base.dev->pdev,
+                                       ppgtt->pd_dma_addr[i],
+                                       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+
+                        for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
+                                dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
+                                if (addr)
+                                        pci_unmap_page(ppgtt->base.dev->pdev,
+                                                       addr,
+                                                       PAGE_SIZE,
+                                                       PCI_DMA_BIDIRECTIONAL);
+
+                        }
+                }
+                kfree(ppgtt->gen8_pt_dma_addr[i]);
+        }
+
+        __free_pages(ppgtt->gen8_pt_pages, get_order(ppgtt->num_pt_pages << PAGE_SHIFT));
+        __free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));
+}
+
+/**
+ * GEN8 legacy ppgtt programming is accomplished through 4 PDP registers with a
+ * net effect resembling a 2-level page table in normal x86 terms. Each PDP
+ * represents 1GB of memory
+ * 4 * 512 * 512 * 4096 = 4GB legacy 32b address space.
+ *
+ * TODO: Do something with the size parameter
+ **/
+static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
+{
+        struct page *pt_pages;
+        int i, j, ret = -ENOMEM;
+        const int max_pdp = DIV_ROUND_UP(size, 1 << 30);
+        const int num_pt_pages = GEN8_PDES_PER_PAGE * max_pdp;
+
+        if (size % (1<<30))
+                DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size);
+
+        /* FIXME: split allocation into smaller pieces. For now we only ever do
+         * this once, but with full PPGTT, the multiple contiguous allocations
+         * will be bad.
+         */
+        ppgtt->pd_pages = alloc_pages(GFP_KERNEL, get_order(max_pdp << PAGE_SHIFT));
+        if (!ppgtt->pd_pages)
+                return -ENOMEM;
+
+        pt_pages = alloc_pages(GFP_KERNEL, get_order(num_pt_pages << PAGE_SHIFT));
+        if (!pt_pages) {
+                __free_pages(ppgtt->pd_pages, get_order(max_pdp << PAGE_SHIFT));
+                return -ENOMEM;
+        }
+
+        ppgtt->gen8_pt_pages = pt_pages;
+        ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT);
+        ppgtt->num_pt_pages = 1 << get_order(num_pt_pages << PAGE_SHIFT);
+        ppgtt->num_pd_entries = max_pdp * GEN8_PDES_PER_PAGE;
+        ppgtt->enable = gen8_ppgtt_enable;
+        ppgtt->base.clear_range = gen8_ppgtt_clear_range;
+        ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
+        ppgtt->base.cleanup = gen8_ppgtt_cleanup;
+
+        BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS);
+
+        /*
+         * - Create a mapping for the page directories.
+         * - For each page directory:
+         *      allocate space for page table mappings.
+         *      map each page table
+         */
+        for (i = 0; i < max_pdp; i++) {
+                dma_addr_t temp;
+                temp = pci_map_page(ppgtt->base.dev->pdev,
+                                    &ppgtt->pd_pages[i], 0,
+                                    PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+                if (pci_dma_mapping_error(ppgtt->base.dev->pdev, temp))
+                        goto err_out;
+
+                ppgtt->pd_dma_addr[i] = temp;
+
+                ppgtt->gen8_pt_dma_addr[i] = kmalloc(sizeof(dma_addr_t) * GEN8_PDES_PER_PAGE, GFP_KERNEL);
+                if (!ppgtt->gen8_pt_dma_addr[i])
+                        goto err_out;
+
+                for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
+                        struct page *p = &pt_pages[i * GEN8_PDES_PER_PAGE + j];
+                        temp = pci_map_page(ppgtt->base.dev->pdev,
+                                            p, 0, PAGE_SIZE,
+                                            PCI_DMA_BIDIRECTIONAL);
+
+                        if (pci_dma_mapping_error(ppgtt->base.dev->pdev, temp))
+                                goto err_out;
+
+                        ppgtt->gen8_pt_dma_addr[i][j] = temp;
+                }
+        }
+
+        /* For now, the PPGTT helper functions all require that the PDEs are
+         * plugged in correctly. So we do that now/here. For aliasing PPGTT, we
+         * will never need to touch the PDEs again */
+        for (i = 0; i < max_pdp; i++) {
+                gen8_ppgtt_pde_t *pd_vaddr;
+                pd_vaddr = kmap_atomic(&ppgtt->pd_pages[i]);
+                for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
+                        dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
+                        pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
+                                                      I915_CACHE_LLC);
+                }
+                kunmap_atomic(pd_vaddr);
+        }
+
+        ppgtt->base.clear_range(&ppgtt->base, 0,
+                                ppgtt->num_pd_entries * GEN8_PTES_PER_PAGE,
+                                true);
+
+        DRM_DEBUG_DRIVER("Allocated %d pages for page directories (%d wasted)\n",
+                         ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);
+        DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%lld wasted)\n",
+                         ppgtt->num_pt_pages,
+                         (ppgtt->num_pt_pages - num_pt_pages) +
+                         size % (1<<30));
+        return 0;
+
+err_out:
+        ppgtt->base.cleanup(&ppgtt->base);
+        return ret;
+}
+
 static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
 {
         struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
@@ -410,6 +698,8 @@ static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
 
         if (INTEL_INFO(dev)->gen < 8)
                 ret = gen6_ppgtt_init(ppgtt);
+        else if (IS_GEN8(dev))
+                ret = gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
         else
                 BUG();
 
@@ -573,6 +863,57 @@ int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
         return 0;
 }
 
+static inline void gen8_set_pte(void __iomem *addr, gen8_gtt_pte_t pte)
+{
+#ifdef writeq
+        writeq(pte, addr);
+#else
+        iowrite32((u32)pte, addr);
+        iowrite32(pte >> 32, addr + 4);
+#endif
+}
+
+static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
+                                     struct sg_table *st,
+                                     unsigned int first_entry,
+                                     enum i915_cache_level level)
+{
+        struct drm_i915_private *dev_priv = vm->dev->dev_private;
+        gen8_gtt_pte_t __iomem *gtt_entries =
+                (gen8_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
+        int i = 0;
+        struct sg_page_iter sg_iter;
+        dma_addr_t addr;
+
+        for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
+                addr = sg_dma_address(sg_iter.sg) +
+                        (sg_iter.sg_pgoffset << PAGE_SHIFT);
+                gen8_set_pte(&gtt_entries[i],
+                             gen8_pte_encode(addr, level, true));
+                i++;
+        }
+
+        /*
+         * XXX: This serves as a posting read to make sure that the PTE has
+         * actually been updated. There is some concern that even though
+         * registers and PTEs are within the same BAR that they are potentially
+         * of NUMA access patterns. Therefore, even with the way we assume
+         * hardware should work, we must keep this posting read for paranoia.
+         */
+        if (i != 0)
+                WARN_ON(readq(&gtt_entries[i-1])
+                        != gen8_pte_encode(addr, level, true));
+
+#if 0 /* TODO: Still needed on GEN8? */
+        /* This next bit makes the above posting read even more important. We
+         * want to flush the TLBs only after we're certain all the PTE updates
+         * have finished.
+         */
+        I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
+        POSTING_READ(GFX_FLSH_CNTL_GEN6);
+#endif
+}
+
 /*
  * Binds an object into the global gtt with the specified cache level. The object
  * will be accessible to the GPU via commands whose operands reference offsets
@@ -615,6 +956,30 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
         POSTING_READ(GFX_FLSH_CNTL_GEN6);
 }
 
+static void gen8_ggtt_clear_range(struct i915_address_space *vm,
+                                  unsigned int first_entry,
+                                  unsigned int num_entries,
+                                  bool use_scratch)
+{
+        struct drm_i915_private *dev_priv = vm->dev->dev_private;
+        gen8_gtt_pte_t scratch_pte, __iomem *gtt_base =
+                (gen8_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
+        const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
+        int i;
+
+        if (WARN(num_entries > max_entries,
+                 "First entry = %d; Num entries = %d (max=%d)\n",
+                 first_entry, num_entries, max_entries))
+                num_entries = max_entries;
+
+        scratch_pte = gen8_pte_encode(vm->scratch.addr,
+                                      I915_CACHE_LLC,
+                                      use_scratch);
+        for (i = 0; i < num_entries; i++)
+                gen8_set_pte(&gtt_base[i], scratch_pte);
+        readl(gtt_base);
+}
+
 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
                                   unsigned int first_entry,
                                   unsigned int num_entries,
@@ -638,7 +1003,6 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
         readl(gtt_base);
 }
 
-
 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
                                      struct sg_table *st,
                                      unsigned int pg_start,
@@ -720,6 +1084,7 @@ static void i915_gtt_color_adjust(struct drm_mm_node *node,
                         *end -= 4096;
         }
 }
+
 void i915_gem_setup_global_gtt(struct drm_device *dev,
                                unsigned long start,
                                unsigned long mappable_end,
@@ -816,7 +1181,8 @@ void i915_gem_init_global_gtt(struct drm_device *dev)
 
                 DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
                 drm_mm_takedown(&dev_priv->gtt.base.mm);
-                gtt_size += GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
+                if (INTEL_INFO(dev)->gen < 8)
+                        gtt_size += GEN6_PPGTT_PD_ENTRIES*PAGE_SIZE;
         }
         i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
 }
@@ -866,6 +1232,20 @@ static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
         return snb_gmch_ctl << 20;
 }
 
+static inline unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
+{
+        bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
+        bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
+        if (bdw_gmch_ctl)
+                bdw_gmch_ctl = 1 << bdw_gmch_ctl;
+        if (bdw_gmch_ctl > 4) {
+                WARN_ON(!i915_preliminary_hw_support);
+                return 4<<20;
+        }
+
+        return bdw_gmch_ctl << 20;
+}
+
 static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
 {
         snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
@@ -873,6 +1253,108 @@ static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
         return snb_gmch_ctl << 25; /* 32 MB units */
 }
 
+static inline size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
+{
+        bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
+        bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
+        return bdw_gmch_ctl << 25; /* 32 MB units */
+}
+
+static int ggtt_probe_common(struct drm_device *dev,
+                             size_t gtt_size)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        phys_addr_t gtt_bus_addr;
+        int ret;
+
+        /* For Modern GENs the PTEs and register space are split in the BAR */
+        gtt_bus_addr = pci_resource_start(dev->pdev, 0) +
+                (pci_resource_len(dev->pdev, 0) / 2);
+
+        dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size);
+        if (!dev_priv->gtt.gsm) {
+                DRM_ERROR("Failed to map the gtt page table\n");
+                return -ENOMEM;
+        }
+
+        ret = setup_scratch_page(dev);
+        if (ret) {
+                DRM_ERROR("Scratch setup failed\n");
+                /* iounmap will also get called at remove, but meh */
+                iounmap(dev_priv->gtt.gsm);
+        }
+
+        return ret;
+}
+
+/* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
+ * bits. When using advanced contexts each context stores its own PAT, but
+ * writing this data shouldn't be harmful even in those cases. */
+static void gen8_setup_private_ppat(struct drm_i915_private *dev_priv)
+{
+#define GEN8_PPAT_UC            (0<<0)
+#define GEN8_PPAT_WC            (1<<0)
+#define GEN8_PPAT_WT            (2<<0)
+#define GEN8_PPAT_WB            (3<<0)
+#define GEN8_PPAT_ELLC_OVERRIDE (0<<2)
+/* FIXME(BDW): Bspec is completely confused about cache control bits. */
+#define GEN8_PPAT_LLC           (1<<2)
+#define GEN8_PPAT_LLCELLC       (2<<2)
+#define GEN8_PPAT_LLCeLLC       (3<<2)
+#define GEN8_PPAT_AGE(x)        (x<<4)
+#define GEN8_PPAT(i, x) ((uint64_t) (x) << ((i) * 8))
+        uint64_t pat;
+
+        pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC)     | /* for normal objects, no eLLC */
+              GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
+              GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
+              GEN8_PPAT(3, GEN8_PPAT_UC)                     | /* Uncached objects, mostly for scanout */
+              GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
+              GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
+              GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
+              GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
+
+        /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
+         * write would work. */
+        I915_WRITE(GEN8_PRIVATE_PAT, pat);
+        I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
+}
+
+static int gen8_gmch_probe(struct drm_device *dev,
+                           size_t *gtt_total,
+                           size_t *stolen,
+                           phys_addr_t *mappable_base,
+                           unsigned long *mappable_end)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned int gtt_size;
+        u16 snb_gmch_ctl;
+        int ret;
+
+        /* TODO: We're not aware of mappable constraints on gen8 yet */
+        *mappable_base = pci_resource_start(dev->pdev, 2);
+        *mappable_end = pci_resource_len(dev->pdev, 2);
+
+        if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
+                pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
+
+        pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
+
+        *stolen = gen8_get_stolen_size(snb_gmch_ctl);
+
+        gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
+        *gtt_total = (gtt_size / sizeof(gen8_gtt_pte_t)) << PAGE_SHIFT;
+
+        gen8_setup_private_ppat(dev_priv);
+
+        ret = ggtt_probe_common(dev, gtt_size);
+
+        dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;
+        dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;
+
+        return ret;
+}
+
 static int gen6_gmch_probe(struct drm_device *dev,
                            size_t *gtt_total,
                            size_t *stolen,
@@ -880,7 +1362,6 @@ static int gen6_gmch_probe(struct drm_device *dev,
                            unsigned long *mappable_end)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        phys_addr_t gtt_bus_addr;
         unsigned int gtt_size;
         u16 snb_gmch_ctl;
         int ret;
@@ -900,24 +1381,13 @@ static int gen6_gmch_probe(struct drm_device *dev,
         if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
                 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
         pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
-        gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
 
         *stolen = gen6_get_stolen_size(snb_gmch_ctl);
-        *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
-
-        /* For Modern GENs the PTEs and register space are split in the BAR */
-        gtt_bus_addr = pci_resource_start(dev->pdev, 0) +
-                (pci_resource_len(dev->pdev, 0) / 2);
 
-        dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size);
-        if (!dev_priv->gtt.gsm) {
-                DRM_ERROR("Failed to map the gtt page table\n");
-                return -ENOMEM;
-        }
+        gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
+        *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
 
-        ret = setup_scratch_page(dev);
-        if (ret)
-                DRM_ERROR("Scratch setup failed\n");
+        ret = ggtt_probe_common(dev, gtt_size);
 
         dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
         dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
@@ -971,7 +1441,7 @@ int i915_gem_gtt_init(struct drm_device *dev)
         if (INTEL_INFO(dev)->gen <= 5) {
                 gtt->gtt_probe = i915_gmch_probe;
                 gtt->base.cleanup = i915_gmch_remove;
-        } else {
+        } else if (INTEL_INFO(dev)->gen < 8) {
                 gtt->gtt_probe = gen6_gmch_probe;
                 gtt->base.cleanup = gen6_gmch_remove;
                 if (IS_HASWELL(dev) && dev_priv->ellc_size)
@@ -984,6 +1454,9 @@ int i915_gem_gtt_init(struct drm_device *dev)
                         gtt->base.pte_encode = ivb_pte_encode;
                 else
                         gtt->base.pte_encode = snb_pte_encode;
+        } else {
+                dev_priv->gtt.gtt_probe = gen8_gmch_probe;
+                dev_priv->gtt.base.cleanup = gen6_gmch_remove;
         }
 
         ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index a8bb213da79f..79dcb8f896c6 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -624,6 +624,7 @@ static void i915_gem_record_fences(struct drm_device *dev,
 
         /* Fences */
         switch (INTEL_INFO(dev)->gen) {
+        case 8:
         case 7:
         case 6:
                 for (i = 0; i < dev_priv->num_fence_regs; i++)
@@ -1044,6 +1045,7 @@ void i915_get_extra_instdone(struct drm_device *dev, uint32_t *instdone)
         default:
                 WARN_ONCE(1, "Unsupported platform\n");
         case 7:
+        case 8:
                 instdone[0] = I915_READ(GEN7_INSTDONE_1);
                 instdone[1] = I915_READ(GEN7_SC_INSTDONE);
                 instdone[2] = I915_READ(GEN7_SAMPLER_INSTDONE);
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 19949e8b36c5..931ee5d8cdb1 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -270,6 +270,21 @@ static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
         }
 }
 
+static void broadwell_set_fifo_underrun_reporting(struct drm_device *dev,
+                                                  enum pipe pipe, bool enable)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        assert_spin_locked(&dev_priv->irq_lock);
+
+        if (enable)
+                dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_FIFO_UNDERRUN;
+        else
+                dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_FIFO_UNDERRUN;
+        I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
+        POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
+}
+
 /**
  * ibx_display_interrupt_update - update SDEIMR
  * @dev_priv: driver private
@@ -382,6 +397,8 @@ bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
                 ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
         else if (IS_GEN7(dev))
                 ivybridge_set_fifo_underrun_reporting(dev, pipe, enable);
+        else if (IS_GEN8(dev))
+                broadwell_set_fifo_underrun_reporting(dev, pipe, enable);
 
 done:
         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
@@ -600,35 +617,40 @@ static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
         return I915_READ(reg);
 }
 
-static bool intel_pipe_in_vblank(struct drm_device *dev, enum pipe pipe)
+/* raw reads, only for fast reads of display block, no need for forcewake etc. */
+#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
+#define __raw_i915_read16(dev_priv__, reg__) readw((dev_priv__)->regs + (reg__))
+
+static bool intel_pipe_in_vblank_locked(struct drm_device *dev, enum pipe pipe)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t status;
+        int reg;
 
         if (IS_VALLEYVIEW(dev)) {
                 status = pipe == PIPE_A ?
                         I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT :
                         I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
 
-                return I915_READ(VLV_ISR) & status;
+                reg = VLV_ISR;
         } else if (IS_GEN2(dev)) {
                 status = pipe == PIPE_A ?
                         I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT :
                         I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
 
-                return I915_READ16(ISR) & status;
+                reg = ISR;
         } else if (INTEL_INFO(dev)->gen < 5) {
                 status = pipe == PIPE_A ?
                         I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT :
                         I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
 
-                return I915_READ(ISR) & status;
+                reg = ISR;
         } else if (INTEL_INFO(dev)->gen < 7) {
                 status = pipe == PIPE_A ?
                         DE_PIPEA_VBLANK :
                         DE_PIPEB_VBLANK;
 
-                return I915_READ(DEISR) & status;
+                reg = DEISR;
         } else {
                 switch (pipe) {
                 default:
@@ -643,12 +665,17 @@ static bool intel_pipe_in_vblank(struct drm_device *dev, enum pipe pipe)
                         break;
                 }
 
-                return I915_READ(DEISR) & status;
+                reg = DEISR;
         }
+
+        if (IS_GEN2(dev))
+                return __raw_i915_read16(dev_priv, reg) & status;
+        else
+                return __raw_i915_read32(dev_priv, reg) & status;
 }
 
 static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
-                             int *vpos, int *hpos)
+                             int *vpos, int *hpos, ktime_t *stime, ktime_t *etime)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
@@ -658,6 +685,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
         int vbl_start, vbl_end, htotal, vtotal;
         bool in_vbl = true;
         int ret = 0;
+        unsigned long irqflags;
 
         if (!intel_crtc->active) {
                 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
@@ -672,14 +700,27 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
 
         ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
 
+        /*
+         * Lock uncore.lock, as we will do multiple timing critical raw
+         * register reads, potentially with preemption disabled, so the
+         * following code must not block on uncore.lock.
+         */
+        spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
+        
+        /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
+
+        /* Get optional system timestamp before query. */
+        if (stime)
+                *stime = ktime_get();
+
         if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
                 /* No obvious pixelcount register. Only query vertical
                  * scanout position from Display scan line register.
                  */
                 if (IS_GEN2(dev))
-                        position = I915_READ(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
+                        position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
                 else
-                        position = I915_READ(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
+                        position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
 
                 /*
                  * The scanline counter increments at the leading edge
@@ -688,7 +729,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
                  * to get a more accurate picture whether we're in vblank
                  * or not.
                  */
-                in_vbl = intel_pipe_in_vblank(dev, pipe);
+                in_vbl = intel_pipe_in_vblank_locked(dev, pipe);
                 if ((in_vbl && position == vbl_start - 1) ||
                     (!in_vbl && position == vbl_end - 1))
                         position = (position + 1) % vtotal;
@@ -697,7 +738,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
                  * We can split this into vertical and horizontal
                  * scanout position.
                  */
-                position = (I915_READ(PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
+                position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
 
                 /* convert to pixel counts */
                 vbl_start *= htotal;
@@ -705,6 +746,14 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
                 vtotal *= htotal;
         }
 
+        /* Get optional system timestamp after query. */
+        if (etime)
+                *etime = ktime_get();
+
+        /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
+
+        spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
+
         in_vbl = position >= vbl_start && position < vbl_end;
 
         /*
@@ -1038,7 +1087,7 @@ static void ivybridge_parity_work(struct work_struct *work)
                 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
                 parity_event[5] = NULL;
 
-                kobject_uevent_env(&dev_priv->dev->primary->kdev.kobj,
+                kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
                                    KOBJ_CHANGE, parity_event);
 
                 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
@@ -1117,6 +1166,56 @@ static void snb_gt_irq_handler(struct drm_device *dev,
                 ivybridge_parity_error_irq_handler(dev, gt_iir);
 }
 
+static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
+                                       struct drm_i915_private *dev_priv,
+                                       u32 master_ctl)
+{
+        u32 rcs, bcs, vcs;
+        uint32_t tmp = 0;
+        irqreturn_t ret = IRQ_NONE;
+
+        if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
+                tmp = I915_READ(GEN8_GT_IIR(0));
+                if (tmp) {
+                        ret = IRQ_HANDLED;
+                        rcs = tmp >> GEN8_RCS_IRQ_SHIFT;
+                        bcs = tmp >> GEN8_BCS_IRQ_SHIFT;
+                        if (rcs & GT_RENDER_USER_INTERRUPT)
+                                notify_ring(dev, &dev_priv->ring[RCS]);
+                        if (bcs & GT_RENDER_USER_INTERRUPT)
+                                notify_ring(dev, &dev_priv->ring[BCS]);
+                        I915_WRITE(GEN8_GT_IIR(0), tmp);
+                } else
+                        DRM_ERROR("The master control interrupt lied (GT0)!\n");
+        }
+
+        if (master_ctl & GEN8_GT_VCS1_IRQ) {
+                tmp = I915_READ(GEN8_GT_IIR(1));
+                if (tmp) {
+                        ret = IRQ_HANDLED;
+                        vcs = tmp >> GEN8_VCS1_IRQ_SHIFT;
+                        if (vcs & GT_RENDER_USER_INTERRUPT)
+                                notify_ring(dev, &dev_priv->ring[VCS]);
+                        I915_WRITE(GEN8_GT_IIR(1), tmp);
+                } else
+                        DRM_ERROR("The master control interrupt lied (GT1)!\n");
+        }
+
+        if (master_ctl & GEN8_GT_VECS_IRQ) {
+                tmp = I915_READ(GEN8_GT_IIR(3));
+                if (tmp) {
+                        ret = IRQ_HANDLED;
+                        vcs = tmp >> GEN8_VECS_IRQ_SHIFT;
+                        if (vcs & GT_RENDER_USER_INTERRUPT)
+                                notify_ring(dev, &dev_priv->ring[VECS]);
+                        I915_WRITE(GEN8_GT_IIR(3), tmp);
+                } else
+                        DRM_ERROR("The master control interrupt lied (GT3)!\n");
+        }
+
+        return ret;
+}
+
 #define HPD_STORM_DETECT_PERIOD 1000
 #define HPD_STORM_THRESHOLD 5
 
@@ -1351,7 +1450,7 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
                 for_each_pipe(pipe) {
-                        if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS)
+                        if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
                                 drm_handle_vblank(dev, pipe);
 
                         if (pipe_stats[pipe] & PLANE_FLIPDONE_INT_STATUS_VLV) {
@@ -1690,6 +1789,117 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
         return ret;
 }
 
+static irqreturn_t gen8_irq_handler(int irq, void *arg)
+{
+        struct drm_device *dev = arg;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 master_ctl;
+        irqreturn_t ret = IRQ_NONE;
+        uint32_t tmp = 0;
+        enum pipe pipe;
+
+        atomic_inc(&dev_priv->irq_received);
+
+        master_ctl = I915_READ(GEN8_MASTER_IRQ);
+        master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
+        if (!master_ctl)
+                return IRQ_NONE;
+
+        I915_WRITE(GEN8_MASTER_IRQ, 0);
+        POSTING_READ(GEN8_MASTER_IRQ);
+
+        ret = gen8_gt_irq_handler(dev, dev_priv, master_ctl);
+
+        if (master_ctl & GEN8_DE_MISC_IRQ) {
+                tmp = I915_READ(GEN8_DE_MISC_IIR);
+                if (tmp & GEN8_DE_MISC_GSE)
+                        intel_opregion_asle_intr(dev);
+                else if (tmp)
+                        DRM_ERROR("Unexpected DE Misc interrupt\n");
+                else
+                        DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
+
+                if (tmp) {
+                        I915_WRITE(GEN8_DE_MISC_IIR, tmp);
+                        ret = IRQ_HANDLED;
+                }
+        }
+
+        if (master_ctl & GEN8_DE_PORT_IRQ) {
+                tmp = I915_READ(GEN8_DE_PORT_IIR);
+                if (tmp & GEN8_AUX_CHANNEL_A)
+                        dp_aux_irq_handler(dev);
+                else if (tmp)
+                        DRM_ERROR("Unexpected DE Port interrupt\n");
+                else
+                        DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
+
+                if (tmp) {
+                        I915_WRITE(GEN8_DE_PORT_IIR, tmp);
+                        ret = IRQ_HANDLED;
+                }
+        }
+
+        for_each_pipe(pipe) {
+                uint32_t pipe_iir;
+
+                if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
+                        continue;
+
+                pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
+                if (pipe_iir & GEN8_PIPE_VBLANK)
+                        drm_handle_vblank(dev, pipe);
+
+                if (pipe_iir & GEN8_PIPE_FLIP_DONE) {
+                        intel_prepare_page_flip(dev, pipe);
+                        intel_finish_page_flip_plane(dev, pipe);
+                }
+
+                if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
+                        hsw_pipe_crc_irq_handler(dev, pipe);
+
+                if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN) {
+                        if (intel_set_cpu_fifo_underrun_reporting(dev, pipe,
+                                                                  false))
+                                DRM_DEBUG_DRIVER("Pipe %c FIFO underrun\n",
+                                                 pipe_name(pipe));
+                }
+
+                if (pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS) {
+                        DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
+                                  pipe_name(pipe),
+                                  pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
+                }
+
+                if (pipe_iir) {
+                        ret = IRQ_HANDLED;
+                        I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
+                } else
+                        DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
+        }
+
+        if (!HAS_PCH_NOP(dev) && master_ctl & GEN8_DE_PCH_IRQ) {
+                /*
+                 * FIXME(BDW): Assume for now that the new interrupt handling
+                 * scheme also closed the SDE interrupt handling race we've seen
+                 * on older pch-split platforms. But this needs testing.
+                 */
+                u32 pch_iir = I915_READ(SDEIIR);
+
+                cpt_irq_handler(dev, pch_iir);
+
+                if (pch_iir) {
+                        I915_WRITE(SDEIIR, pch_iir);
+                        ret = IRQ_HANDLED;
+                }
+        }
+
+        I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
+        POSTING_READ(GEN8_MASTER_IRQ);
+
+        return ret;
+}
+
 static void i915_error_wake_up(struct drm_i915_private *dev_priv,
                                bool reset_completed)
 {
@@ -1737,7 +1947,7 @@ static void i915_error_work_func(struct work_struct *work)
         char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
         int ret;
 
-        kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE, error_event);
+        kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
 
         /*
          * Note that there's only one work item which does gpu resets, so we
@@ -1751,7 +1961,7 @@ static void i915_error_work_func(struct work_struct *work)
          */
         if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
                 DRM_DEBUG_DRIVER("resetting chip\n");
-                kobject_uevent_env(&dev->primary->kdev.kobj, KOBJ_CHANGE,
+                kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
                                    reset_event);
 
                 /*
@@ -1778,7 +1988,7 @@ static void i915_error_work_func(struct work_struct *work)
                         smp_mb__before_atomic_inc();
                         atomic_inc(&dev_priv->gpu_error.reset_counter);
 
-                        kobject_uevent_env(&dev->primary->kdev.kobj,
+                        kobject_uevent_env(&dev->primary->kdev->kobj,
                                            KOBJ_CHANGE, reset_done_event);
                 } else {
                         atomic_set_mask(I915_WEDGED, &error->reset_counter);
@@ -2043,6 +2253,22 @@ static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
         return 0;
 }
 
+static int gen8_enable_vblank(struct drm_device *dev, int pipe)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long irqflags;
+
+        if (!i915_pipe_enabled(dev, pipe))
+                return -EINVAL;
+
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+        dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
+        I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
+        POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+        return 0;
+}
+
 /* Called from drm generic code, passed 'crtc' which
  * we use as a pipe index
  */
@@ -2091,6 +2317,21 @@ static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
+static void gen8_disable_vblank(struct drm_device *dev, int pipe)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long irqflags;
+
+        if (!i915_pipe_enabled(dev, pipe))
+                return;
+
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+        dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
+        I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
+        POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+}
+
 static u32
 ring_last_seqno(struct intel_ring_buffer *ring)
 {
@@ -2425,6 +2666,53 @@ static void valleyview_irq_preinstall(struct drm_device *dev)
         POSTING_READ(VLV_IER);
 }
 
+static void gen8_irq_preinstall(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int pipe;
+
+        atomic_set(&dev_priv->irq_received, 0);
+
+        I915_WRITE(GEN8_MASTER_IRQ, 0);
+        POSTING_READ(GEN8_MASTER_IRQ);
+
+        /* IIR can theoretically queue up two events. Be paranoid */
+#define GEN8_IRQ_INIT_NDX(type, which) do { \
+                I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
+                POSTING_READ(GEN8_##type##_IMR(which)); \
+                I915_WRITE(GEN8_##type##_IER(which), 0); \
+                I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
+                POSTING_READ(GEN8_##type##_IIR(which)); \
+                I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
+        } while (0)
+
+#define GEN8_IRQ_INIT(type) do { \
+                I915_WRITE(GEN8_##type##_IMR, 0xffffffff); \
+                POSTING_READ(GEN8_##type##_IMR); \
+                I915_WRITE(GEN8_##type##_IER, 0); \
+                I915_WRITE(GEN8_##type##_IIR, 0xffffffff); \
+                POSTING_READ(GEN8_##type##_IIR); \
+                I915_WRITE(GEN8_##type##_IIR, 0xffffffff); \
+        } while (0)
+
+        GEN8_IRQ_INIT_NDX(GT, 0);
+        GEN8_IRQ_INIT_NDX(GT, 1);
+        GEN8_IRQ_INIT_NDX(GT, 2);
+        GEN8_IRQ_INIT_NDX(GT, 3);
+
+        for_each_pipe(pipe) {
+                GEN8_IRQ_INIT_NDX(DE_PIPE, pipe);
+        }
+
+        GEN8_IRQ_INIT(DE_PORT);
+        GEN8_IRQ_INIT(DE_MISC);
+        GEN8_IRQ_INIT(PCU);
+#undef GEN8_IRQ_INIT
+#undef GEN8_IRQ_INIT_NDX
+
+        POSTING_READ(GEN8_PCU_IIR);
+}
+
 static void ibx_hpd_irq_setup(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
@@ -2630,6 +2918,117 @@ static int valleyview_irq_postinstall(struct drm_device *dev)
         return 0;
 }
 
+static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
+{
+        int i;
+
+        /* These are interrupts we'll toggle with the ring mask register */
+        uint32_t gt_interrupts[] = {
+                GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
+                        GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
+                        GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
+                GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
+                        GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
+                0,
+                GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT
+                };
+
+        for (i = 0; i < ARRAY_SIZE(gt_interrupts); i++) {
+                u32 tmp = I915_READ(GEN8_GT_IIR(i));
+                if (tmp)
+                        DRM_ERROR("Interrupt (%d) should have been masked in pre-install 0x%08x\n",
+                                  i, tmp);
+                I915_WRITE(GEN8_GT_IMR(i), ~gt_interrupts[i]);
+                I915_WRITE(GEN8_GT_IER(i), gt_interrupts[i]);
+        }
+        POSTING_READ(GEN8_GT_IER(0));
+}
+
+static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
+{
+        struct drm_device *dev = dev_priv->dev;
+        uint32_t de_pipe_masked = GEN8_PIPE_FLIP_DONE |
+                GEN8_PIPE_CDCLK_CRC_DONE |
+                GEN8_PIPE_FIFO_UNDERRUN |
+                GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
+        uint32_t de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK;
+        int pipe;
+        dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
+        dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
+        dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
+
+        for_each_pipe(pipe) {
+                u32 tmp = I915_READ(GEN8_DE_PIPE_IIR(pipe));
+                if (tmp)
+                        DRM_ERROR("Interrupt (%d) should have been masked in pre-install 0x%08x\n",
+                                  pipe, tmp);
+                I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
+                I915_WRITE(GEN8_DE_PIPE_IER(pipe), de_pipe_enables);
+        }
+        POSTING_READ(GEN8_DE_PIPE_ISR(0));
+
+        I915_WRITE(GEN8_DE_PORT_IMR, ~GEN8_AUX_CHANNEL_A);
+        I915_WRITE(GEN8_DE_PORT_IER, GEN8_AUX_CHANNEL_A);
+        POSTING_READ(GEN8_DE_PORT_IER);
+}
+
+static int gen8_irq_postinstall(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        gen8_gt_irq_postinstall(dev_priv);
+        gen8_de_irq_postinstall(dev_priv);
+
+        ibx_irq_postinstall(dev);
+
+        I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
+        POSTING_READ(GEN8_MASTER_IRQ);
+
+        return 0;
+}
+
+static void gen8_irq_uninstall(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int pipe;
+
+        if (!dev_priv)
+                return;
+
+        atomic_set(&dev_priv->irq_received, 0);
+
+        I915_WRITE(GEN8_MASTER_IRQ, 0);
+
+#define GEN8_IRQ_FINI_NDX(type, which) do { \
+                I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
+                I915_WRITE(GEN8_##type##_IER(which), 0); \
+                I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
+        } while (0)
+
+#define GEN8_IRQ_FINI(type) do { \
+                I915_WRITE(GEN8_##type##_IMR, 0xffffffff); \
+                I915_WRITE(GEN8_##type##_IER, 0); \
+                I915_WRITE(GEN8_##type##_IIR, 0xffffffff); \
+        } while (0)
+
+        GEN8_IRQ_FINI_NDX(GT, 0);
+        GEN8_IRQ_FINI_NDX(GT, 1);
+        GEN8_IRQ_FINI_NDX(GT, 2);
+        GEN8_IRQ_FINI_NDX(GT, 3);
+
+        for_each_pipe(pipe) {
+                GEN8_IRQ_FINI_NDX(DE_PIPE, pipe);
+        }
+
+        GEN8_IRQ_FINI(DE_PORT);
+        GEN8_IRQ_FINI(DE_MISC);
+        GEN8_IRQ_FINI(PCU);
+#undef GEN8_IRQ_FINI
+#undef GEN8_IRQ_FINI_NDX
+
+        POSTING_READ(GEN8_PCU_IIR);
+}
+
 static void valleyview_irq_uninstall(struct drm_device *dev)
 {
         drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
@@ -3409,6 +3808,14 @@ void intel_irq_init(struct drm_device *dev)
                 dev->driver->enable_vblank = valleyview_enable_vblank;
                 dev->driver->disable_vblank = valleyview_disable_vblank;
                 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
+        } else if (IS_GEN8(dev)) {
+                dev->driver->irq_handler = gen8_irq_handler;
+                dev->driver->irq_preinstall = gen8_irq_preinstall;
+                dev->driver->irq_postinstall = gen8_irq_postinstall;
+                dev->driver->irq_uninstall = gen8_irq_uninstall;
+                dev->driver->enable_vblank = gen8_enable_vblank;
+                dev->driver->disable_vblank = gen8_disable_vblank;
+                dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
         } else if (HAS_PCH_SPLIT(dev)) {
                 dev->driver->irq_handler = ironlake_irq_handler;
                 dev->driver->irq_preinstall = ironlake_irq_preinstall;
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index a8a5bcb521c7..849e595ed19d 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -110,6 +110,9 @@
 #define RING_PP_DIR_DCLV(ring)          ((ring)->mmio_base+0x220)
 #define   PP_DIR_DCLV_2G                0xffffffff
 
+#define GEN8_RING_PDP_UDW(ring, n)      ((ring)->mmio_base+0x270 + ((n) * 8 + 4))
+#define GEN8_RING_PDP_LDW(ring, n)      ((ring)->mmio_base+0x270 + (n) * 8)
+
 #define GAM_ECOCHK                      0x4090
 #define   ECOCHK_SNB_BIT                (1<<10)
 #define   HSW_ECOCHK_ARB_PRIO_SOL       (1<<6)
@@ -247,6 +250,7 @@
 #define   MI_BATCH_NON_SECURE_HSW       (1<<13)
 #define MI_BATCH_BUFFER_START   MI_INSTR(0x31, 0)
 #define   MI_BATCH_GTT              (2<<6) /* aliased with (1<<7) on gen4 */
+#define MI_BATCH_BUFFER_START_GEN8      MI_INSTR(0x31, 1)
 #define MI_SEMAPHORE_MBOX       MI_INSTR(0x16, 1) /* gen6+ */
 #define  MI_SEMAPHORE_GLOBAL_GTT    (1<<22)
 #define  MI_SEMAPHORE_UPDATE        (1<<21)
@@ -657,6 +661,9 @@
 #define ARB_MODE                0x04030
 #define   ARB_MODE_SWIZZLE_SNB  (1<<4)
 #define   ARB_MODE_SWIZZLE_IVB  (1<<5)
+#define GAMTARBMODE             0x04a08
+#define   ARB_MODE_BWGTLB_DISABLE (1<<9)
+#define   ARB_MODE_SWIZZLE_BDW  (1<<1)
 #define RENDER_HWS_PGA_GEN7     (0x04080)
 #define RING_FAULT_REG(ring)    (0x4094 + 0x100*(ring)->id)
 #define   RING_FAULT_GTTSEL_MASK (1<<11)
@@ -664,6 +671,7 @@
 #define   RING_FAULT_FAULT_TYPE(x) ((x >> 1) & 0x3)
 #define   RING_FAULT_VALID      (1<<0)
 #define DONE_REG                0x40b0
+#define GEN8_PRIVATE_PAT        0x40e0
 #define BSD_HWS_PGA_GEN7        (0x04180)
 #define BLT_HWS_PGA_GEN7        (0x04280)
 #define VEBOX_HWS_PGA_GEN7      (0x04380)
@@ -743,6 +751,7 @@
 #define   FPGA_DBG_RM_NOCLAIM   (1<<31)
 
 #define DERRMR          0x44050
+/* Note that HBLANK events are reserved on bdw+ */
 #define   DERRMR_PIPEA_SCANLINE         (1<<0)
 #define   DERRMR_PIPEA_PRI_FLIP_DONE    (1<<1)
 #define   DERRMR_PIPEA_SPR_FLIP_DONE    (1<<2)
@@ -776,6 +785,7 @@
 #define _3D_CHICKEN3    0x02090
 #define  _3D_CHICKEN_SF_DISABLE_OBJEND_CULL             (1 << 10)
 #define  _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL          (1 << 5)
+#define  _3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(x)       ((x)<<1)
 
 #define MI_MODE         0x0209c
 # define VS_TIMER_DISPATCH                              (1 << 6)
@@ -1822,6 +1832,9 @@
  * on HSW) - so the final size is 66944 bytes, which rounds to 17 pages.
  */
 #define HSW_CXT_TOTAL_SIZE              (17 * PAGE_SIZE)
+/* Same as Haswell, but 72064 bytes now. */
+#define GEN8_CXT_TOTAL_SIZE             (18 * PAGE_SIZE)
+
 
 #define VLV_CLK_CTL2                    0x101104
 #define   CLK_CTL2_CZCOUNT_30NS_SHIFT   28
@@ -1952,8 +1965,8 @@
 #define BCLRPAT(pipe) _PIPE(pipe, _BCLRPAT_A, _BCLRPAT_B)
 #define VSYNCSHIFT(trans) _TRANSCODER(trans, _VSYNCSHIFT_A, _VSYNCSHIFT_B)
 
-/* HSW eDP PSR registers */
-#define EDP_PSR_BASE(dev)                       0x64800
+/* HSW+ eDP PSR registers */
+#define EDP_PSR_BASE(dev)                       (IS_HASWELL(dev) ? 0x64800 : 0x6f800)
 #define EDP_PSR_CTL(dev)                        (EDP_PSR_BASE(dev) + 0)
 #define   EDP_PSR_ENABLE                        (1<<31)
 #define   EDP_PSR_LINK_DISABLE                  (0<<27)
@@ -2397,6 +2410,21 @@
 
 #define PFIT_AUTO_RATIOS (dev_priv->info->display_mmio_offset + 0x61238)
 
+#define _VLV_BLC_PWM_CTL2_A (dev_priv->info->display_mmio_offset + 0x61250)
+#define _VLV_BLC_PWM_CTL2_B (dev_priv->info->display_mmio_offset + 0x61350)
+#define VLV_BLC_PWM_CTL2(pipe) _PIPE(pipe, _VLV_BLC_PWM_CTL2_A, \
+                                     _VLV_BLC_PWM_CTL2_B)
+
+#define _VLV_BLC_PWM_CTL_A (dev_priv->info->display_mmio_offset + 0x61254)
+#define _VLV_BLC_PWM_CTL_B (dev_priv->info->display_mmio_offset + 0x61354)
+#define VLV_BLC_PWM_CTL(pipe) _PIPE(pipe, _VLV_BLC_PWM_CTL_A, \
+                                    _VLV_BLC_PWM_CTL_B)
+
+#define _VLV_BLC_HIST_CTL_A (dev_priv->info->display_mmio_offset + 0x61260)
+#define _VLV_BLC_HIST_CTL_B (dev_priv->info->display_mmio_offset + 0x61360)
+#define VLV_BLC_HIST_CTL(pipe) _PIPE(pipe, _VLV_BLC_HIST_CTL_A, \
+                                     _VLV_BLC_HIST_CTL_B)
+
 /* Backlight control */
 #define BLC_PWM_CTL2    (dev_priv->info->display_mmio_offset + 0x61250) /* 965+ only */
 #define   BLM_PWM_ENABLE                (1 << 31)
@@ -3228,6 +3256,18 @@
 #define PIPEFRAMEPIXEL(pipe)  _PIPE(pipe, _PIPEAFRAMEPIXEL, _PIPEBFRAMEPIXEL)
 #define PIPESTAT(pipe) _PIPE(pipe, _PIPEASTAT, _PIPEBSTAT)
 
+#define _PIPE_MISC_A                    0x70030
+#define _PIPE_MISC_B                    0x71030
+#define   PIPEMISC_DITHER_BPC_MASK      (7<<5)
+#define   PIPEMISC_DITHER_8_BPC         (0<<5)
+#define   PIPEMISC_DITHER_10_BPC        (1<<5)
+#define   PIPEMISC_DITHER_6_BPC         (2<<5)
+#define   PIPEMISC_DITHER_12_BPC        (3<<5)
+#define   PIPEMISC_DITHER_ENABLE        (1<<4)
+#define   PIPEMISC_DITHER_TYPE_MASK     (3<<2)
+#define   PIPEMISC_DITHER_TYPE_SP       (0<<2)
+#define PIPEMISC(pipe) _PIPE(pipe, _PIPE_MISC_A, _PIPE_MISC_B)
+
 #define VLV_DPFLIPSTAT                          (VLV_DISPLAY_BASE + 0x70028)
 #define   PIPEB_LINE_COMPARE_INT_EN             (1<<29)
 #define   PIPEB_HLINE_INT_EN                    (1<<28)
@@ -3358,6 +3398,7 @@
 #define  WM1_LP_LATENCY_MASK    (0x7f<<24)
 #define  WM1_LP_FBC_MASK        (0xf<<20)
 #define  WM1_LP_FBC_SHIFT       20
+#define  WM1_LP_FBC_SHIFT_BDW   19
 #define  WM1_LP_SR_MASK         (0x7ff<<8)
 #define  WM1_LP_SR_SHIFT        8
 #define  WM1_LP_CURSOR_MASK     (0xff)
@@ -3998,6 +4039,71 @@
 #define GTIIR   0x44018
 #define GTIER   0x4401c
 
+#define GEN8_MASTER_IRQ                 0x44200
+#define  GEN8_MASTER_IRQ_CONTROL        (1<<31)
+#define  GEN8_PCU_IRQ                   (1<<30)
+#define  GEN8_DE_PCH_IRQ                (1<<23)
+#define  GEN8_DE_MISC_IRQ               (1<<22)
+#define  GEN8_DE_PORT_IRQ               (1<<20)
+#define  GEN8_DE_PIPE_C_IRQ             (1<<18)
+#define  GEN8_DE_PIPE_B_IRQ             (1<<17)
+#define  GEN8_DE_PIPE_A_IRQ             (1<<16)
+#define  GEN8_DE_PIPE_IRQ(pipe)         (1<<(16+pipe))
+#define  GEN8_GT_VECS_IRQ               (1<<6)
+#define  GEN8_GT_VCS2_IRQ               (1<<3)
+#define  GEN8_GT_VCS1_IRQ               (1<<2)
+#define  GEN8_GT_BCS_IRQ                (1<<1)
+#define  GEN8_GT_RCS_IRQ                (1<<0)
+
+#define GEN8_GT_ISR(which) (0x44300 + (0x10 * (which)))
+#define GEN8_GT_IMR(which) (0x44304 + (0x10 * (which)))
+#define GEN8_GT_IIR(which) (0x44308 + (0x10 * (which)))
+#define GEN8_GT_IER(which) (0x4430c + (0x10 * (which)))
+
+#define GEN8_BCS_IRQ_SHIFT 16
+#define GEN8_RCS_IRQ_SHIFT 0
+#define GEN8_VCS2_IRQ_SHIFT 16
+#define GEN8_VCS1_IRQ_SHIFT 0
+#define GEN8_VECS_IRQ_SHIFT 0
+
+#define GEN8_DE_PIPE_ISR(pipe) (0x44400 + (0x10 * (pipe)))
+#define GEN8_DE_PIPE_IMR(pipe) (0x44404 + (0x10 * (pipe)))
+#define GEN8_DE_PIPE_IIR(pipe) (0x44408 + (0x10 * (pipe)))
+#define GEN8_DE_PIPE_IER(pipe) (0x4440c + (0x10 * (pipe)))
+#define  GEN8_PIPE_FIFO_UNDERRUN        (1 << 31)
+#define  GEN8_PIPE_CDCLK_CRC_ERROR      (1 << 29)
+#define  GEN8_PIPE_CDCLK_CRC_DONE       (1 << 28)
+#define  GEN8_PIPE_CURSOR_FAULT         (1 << 10)
+#define  GEN8_PIPE_SPRITE_FAULT         (1 << 9)
+#define  GEN8_PIPE_PRIMARY_FAULT        (1 << 8)
+#define  GEN8_PIPE_SPRITE_FLIP_DONE     (1 << 5)
+#define  GEN8_PIPE_FLIP_DONE            (1 << 4)
+#define  GEN8_PIPE_SCAN_LINE_EVENT      (1 << 2)
+#define  GEN8_PIPE_VSYNC                (1 << 1)
+#define  GEN8_PIPE_VBLANK               (1 << 0)
+#define GEN8_DE_PIPE_IRQ_FAULT_ERRORS \
+        (GEN8_PIPE_CURSOR_FAULT | \
+         GEN8_PIPE_SPRITE_FAULT | \
+         GEN8_PIPE_PRIMARY_FAULT)
+
+#define GEN8_DE_PORT_ISR 0x44440
+#define GEN8_DE_PORT_IMR 0x44444
+#define GEN8_DE_PORT_IIR 0x44448
+#define GEN8_DE_PORT_IER 0x4444c
+#define  GEN8_PORT_DP_A_HOTPLUG         (1 << 3)
+#define  GEN8_AUX_CHANNEL_A             (1 << 0)
+
+#define GEN8_DE_MISC_ISR 0x44460
+#define GEN8_DE_MISC_IMR 0x44464
+#define GEN8_DE_MISC_IIR 0x44468
+#define GEN8_DE_MISC_IER 0x4446c
+#define  GEN8_DE_MISC_GSE               (1 << 27)
+
+#define GEN8_PCU_ISR 0x444e0
+#define GEN8_PCU_IMR 0x444e4
+#define GEN8_PCU_IIR 0x444e8
+#define GEN8_PCU_IER 0x444ec
+
 #define ILK_DISPLAY_CHICKEN2    0x42004
 /* Required on all Ironlake and Sandybridge according to the B-Spec. */
 #define  ILK_ELPIN_409_SELECT   (1 << 25)
@@ -4023,8 +4129,14 @@
 # define CHICKEN3_DGMG_DONE_FIX_DISABLE         (1 << 2)
 
 #define CHICKEN_PAR1_1          0x42080
+#define  DPA_MASK_VBLANK_SRD    (1 << 15)
 #define  FORCE_ARB_IDLE_PLANES  (1 << 14)
 
+#define _CHICKEN_PIPESL_1_A     0x420b0
+#define _CHICKEN_PIPESL_1_B     0x420b4
+#define  DPRS_MASK_VBLANK_SRD   (1 << 0)
+#define CHICKEN_PIPESL_1(pipe) _PIPE(pipe, _CHICKEN_PIPESL_1_A, _CHICKEN_PIPESL_1_B)
+
 #define DISP_ARB_CTL    0x45000
 #define  DISP_TILE_SURFACE_SWIZZLING    (1<<13)
 #define  DISP_FBC_WM_DIS                (1<<15)
@@ -4035,6 +4147,8 @@
 /* GEN7 chicken */
 #define GEN7_COMMON_SLICE_CHICKEN1              0x7010
 # define GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC      ((1<<10) | (1<<26))
+#define COMMON_SLICE_CHICKEN2                   0x7014
+# define GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE   (1<<0)
 
 #define GEN7_L3CNTLREG1                         0xB01C
 #define  GEN7_WA_FOR_GEN7_L3_CONTROL                    0x3C4FFF8C
@@ -4863,6 +4977,7 @@
 #define   GEN6_PCODE_WRITE_D_COMP               0x11
 #define   GEN6_ENCODE_RC6_VID(mv)               (((mv) - 245) / 5)
 #define   GEN6_DECODE_RC6_VID(vids)             (((vids) * 5) + 245)
+#define   DISPLAY_IPS_CONTROL                   0x19
 #define GEN6_PCODE_DATA                         0x138128
 #define   GEN6_PCODE_FREQ_IA_RATIO_SHIFT        8
 #define   GEN6_PCODE_FREQ_RING_RATIO_SHIFT      16
@@ -4900,6 +5015,7 @@
 #define GEN7_HALF_SLICE_CHICKEN1        0xe100 /* IVB GT1 + VLV */
 #define GEN7_HALF_SLICE_CHICKEN1_GT2    0xf100
 #define   GEN7_MAX_PS_THREAD_DEP                (8<<12)
+#define   GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE   (1<<10)
 #define   GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE  (1<<3)
 
 #define GEN7_ROW_CHICKEN2               0xe4f4
@@ -4909,6 +5025,10 @@
 #define HSW_ROW_CHICKEN3                0xe49c
 #define  HSW_ROW_CHICKEN3_L3_GLOBAL_ATOMICS_DISABLE    (1 << 6)
 
+#define HALF_SLICE_CHICKEN3             0xe184
+#define   GEN8_CENTROID_PIXEL_OPT_DIS   (1<<8)
+#define   GEN8_SAMPLER_POWER_BYPASS_DIS (1<<1)
+
 #define G4X_AUD_VID_DID                 (dev_priv->info->display_mmio_offset + 0x62020)
 #define INTEL_AUDIO_DEVCL               0x808629FB
 #define INTEL_AUDIO_DEVBLC              0x80862801
@@ -4950,6 +5070,18 @@
                                         CPT_AUD_CNTL_ST_B)
 #define CPT_AUD_CNTRL_ST2               0xE50C0
 
+#define VLV_HDMIW_HDMIEDID_A            (VLV_DISPLAY_BASE + 0x62050)
+#define VLV_HDMIW_HDMIEDID_B            (VLV_DISPLAY_BASE + 0x62150)
+#define VLV_HDMIW_HDMIEDID(pipe) _PIPE(pipe, \
+                                        VLV_HDMIW_HDMIEDID_A, \
+                                        VLV_HDMIW_HDMIEDID_B)
+#define VLV_AUD_CNTL_ST_A               (VLV_DISPLAY_BASE + 0x620B4)
+#define VLV_AUD_CNTL_ST_B               (VLV_DISPLAY_BASE + 0x621B4)
+#define VLV_AUD_CNTL_ST(pipe) _PIPE(pipe, \
+                                        VLV_AUD_CNTL_ST_A, \
+                                        VLV_AUD_CNTL_ST_B)
+#define VLV_AUD_CNTL_ST2                (VLV_DISPLAY_BASE + 0x620C0)
+
 /* These are the 4 32-bit write offset registers for each stream
  * output buffer.  It determines the offset from the
  * 3DSTATE_SO_BUFFERs that the next streamed vertex output goes to.
@@ -4966,6 +5098,12 @@
 #define CPT_AUD_CFG(pipe) _PIPE(pipe, \
                                         CPT_AUD_CONFIG_A, \
                                         CPT_AUD_CONFIG_B)
+#define VLV_AUD_CONFIG_A                (VLV_DISPLAY_BASE + 0x62000)
+#define VLV_AUD_CONFIG_B                (VLV_DISPLAY_BASE + 0x62100)
+#define VLV_AUD_CFG(pipe) _PIPE(pipe, \
+                                        VLV_AUD_CONFIG_A, \
+                                        VLV_AUD_CONFIG_B)
+
 #define   AUD_CONFIG_N_VALUE_INDEX              (1 << 29)
 #define   AUD_CONFIG_N_PROG_ENABLE              (1 << 28)
 #define   AUD_CONFIG_UPPER_N_SHIFT              20
@@ -5108,6 +5246,7 @@
 #define DDI_BUF_CTL_B                           0x64100
 #define DDI_BUF_CTL(port) _PORT(port, DDI_BUF_CTL_A, DDI_BUF_CTL_B)
 #define  DDI_BUF_CTL_ENABLE                     (1<<31)
+/* Haswell */
 #define  DDI_BUF_EMP_400MV_0DB_HSW              (0<<24)   /* Sel0 */
 #define  DDI_BUF_EMP_400MV_3_5DB_HSW            (1<<24)   /* Sel1 */
 #define  DDI_BUF_EMP_400MV_6DB_HSW              (2<<24)   /* Sel2 */
@@ -5117,6 +5256,16 @@
 #define  DDI_BUF_EMP_600MV_6DB_HSW              (6<<24)   /* Sel6 */
 #define  DDI_BUF_EMP_800MV_0DB_HSW              (7<<24)   /* Sel7 */
 #define  DDI_BUF_EMP_800MV_3_5DB_HSW            (8<<24)   /* Sel8 */
+/* Broadwell */
+#define  DDI_BUF_EMP_400MV_0DB_BDW              (0<<24)   /* Sel0 */
+#define  DDI_BUF_EMP_400MV_3_5DB_BDW            (1<<24)   /* Sel1 */
+#define  DDI_BUF_EMP_400MV_6DB_BDW              (2<<24)   /* Sel2 */
+#define  DDI_BUF_EMP_600MV_0DB_BDW              (3<<24)   /* Sel3 */
+#define  DDI_BUF_EMP_600MV_3_5DB_BDW            (4<<24)   /* Sel4 */
+#define  DDI_BUF_EMP_600MV_6DB_BDW              (5<<24)   /* Sel5 */
+#define  DDI_BUF_EMP_800MV_0DB_BDW              (6<<24)   /* Sel6 */
+#define  DDI_BUF_EMP_800MV_3_5DB_BDW            (7<<24)   /* Sel7 */
+#define  DDI_BUF_EMP_1200MV_0DB_BDW             (8<<24)   /* Sel8 */
 #define  DDI_BUF_EMP_MASK                       (0xf<<24)
 #define  DDI_BUF_PORT_REVERSAL                  (1<<16)
 #define  DDI_BUF_IS_IDLE                        (1<<7)
@@ -5226,6 +5375,9 @@
 #define  LCPLL_PLL_LOCK                 (1<<30)
 #define  LCPLL_CLK_FREQ_MASK            (3<<26)
 #define  LCPLL_CLK_FREQ_450             (0<<26)
+#define  LCPLL_CLK_FREQ_54O_BDW         (1<<26)
+#define  LCPLL_CLK_FREQ_337_5_BDW       (2<<26)
+#define  LCPLL_CLK_FREQ_675_BDW         (3<<26)
 #define  LCPLL_CD_CLOCK_DISABLE         (1<<25)
 #define  LCPLL_CD2X_CLOCK_DISABLE       (1<<23)
 #define  LCPLL_POWER_DOWN_ALLOW         (1<<22)
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index a088f1f46bdb..6b8fef7fb3bb 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -192,7 +192,6 @@ static void i915_restore_vga(struct drm_device *dev)
 static void i915_save_display(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
 
         /* Display arbitration control */
         if (INTEL_INFO(dev)->gen <= 4)
@@ -203,30 +202,27 @@ static void i915_save_display(struct drm_device *dev)
         if (!drm_core_check_feature(dev, DRIVER_MODESET))
                 i915_save_display_reg(dev);
 
-        spin_lock_irqsave(&dev_priv->backlight.lock, flags);
-
         /* LVDS state */
         if (HAS_PCH_SPLIT(dev)) {
                 dev_priv->regfile.savePP_CONTROL = I915_READ(PCH_PP_CONTROL);
-                dev_priv->regfile.saveBLC_PWM_CTL = I915_READ(BLC_PWM_PCH_CTL1);
-                dev_priv->regfile.saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_PCH_CTL2);
-                dev_priv->regfile.saveBLC_CPU_PWM_CTL = I915_READ(BLC_PWM_CPU_CTL);
-                dev_priv->regfile.saveBLC_CPU_PWM_CTL2 = I915_READ(BLC_PWM_CPU_CTL2);
                 if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev))
                         dev_priv->regfile.saveLVDS = I915_READ(PCH_LVDS);
+        } else if (IS_VALLEYVIEW(dev)) {
+                dev_priv->regfile.savePP_CONTROL = I915_READ(PP_CONTROL);
+                dev_priv->regfile.savePFIT_PGM_RATIOS = I915_READ(PFIT_PGM_RATIOS);
+
+                dev_priv->regfile.saveBLC_HIST_CTL =
+                        I915_READ(VLV_BLC_HIST_CTL(PIPE_A));
+                dev_priv->regfile.saveBLC_HIST_CTL_B =
+                        I915_READ(VLV_BLC_HIST_CTL(PIPE_B));
         } else {
                 dev_priv->regfile.savePP_CONTROL = I915_READ(PP_CONTROL);
                 dev_priv->regfile.savePFIT_PGM_RATIOS = I915_READ(PFIT_PGM_RATIOS);
-                dev_priv->regfile.saveBLC_PWM_CTL = I915_READ(BLC_PWM_CTL);
                 dev_priv->regfile.saveBLC_HIST_CTL = I915_READ(BLC_HIST_CTL);
-                if (INTEL_INFO(dev)->gen >= 4)
-                        dev_priv->regfile.saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_CTL2);
                 if (IS_MOBILE(dev) && !IS_I830(dev))
                         dev_priv->regfile.saveLVDS = I915_READ(LVDS);
         }
 
-        spin_unlock_irqrestore(&dev_priv->backlight.lock, flags);
-
         if (!IS_I830(dev) && !IS_845G(dev) && !HAS_PCH_SPLIT(dev))
                 dev_priv->regfile.savePFIT_CONTROL = I915_READ(PFIT_CONTROL);
 
@@ -262,7 +258,6 @@ static void i915_restore_display(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 mask = 0xffffffff;
-        unsigned long flags;
 
         /* Display arbitration */
         if (INTEL_INFO(dev)->gen <= 4)
@@ -271,12 +266,6 @@ static void i915_restore_display(struct drm_device *dev)
         if (!drm_core_check_feature(dev, DRIVER_MODESET))
                 i915_restore_display_reg(dev);
 
-        spin_lock_irqsave(&dev_priv->backlight.lock, flags);
-
-        /* LVDS state */
-        if (INTEL_INFO(dev)->gen >= 4 && !HAS_PCH_SPLIT(dev))
-                I915_WRITE(BLC_PWM_CTL2, dev_priv->regfile.saveBLC_PWM_CTL2);
-
         if (drm_core_check_feature(dev, DRIVER_MODESET))
                 mask = ~LVDS_PORT_EN;
 
@@ -289,22 +278,19 @@ static void i915_restore_display(struct drm_device *dev)
                 I915_WRITE(PFIT_CONTROL, dev_priv->regfile.savePFIT_CONTROL);
 
         if (HAS_PCH_SPLIT(dev)) {
-                I915_WRITE(BLC_PWM_PCH_CTL1, dev_priv->regfile.saveBLC_PWM_CTL);
-                I915_WRITE(BLC_PWM_PCH_CTL2, dev_priv->regfile.saveBLC_PWM_CTL2);
-                /* NOTE: BLC_PWM_CPU_CTL must be written after BLC_PWM_CPU_CTL2;
-                 * otherwise we get blank eDP screen after S3 on some machines
-                 */
-                I915_WRITE(BLC_PWM_CPU_CTL2, dev_priv->regfile.saveBLC_CPU_PWM_CTL2);
-                I915_WRITE(BLC_PWM_CPU_CTL, dev_priv->regfile.saveBLC_CPU_PWM_CTL);
                 I915_WRITE(PCH_PP_ON_DELAYS, dev_priv->regfile.savePP_ON_DELAYS);
                 I915_WRITE(PCH_PP_OFF_DELAYS, dev_priv->regfile.savePP_OFF_DELAYS);
                 I915_WRITE(PCH_PP_DIVISOR, dev_priv->regfile.savePP_DIVISOR);
                 I915_WRITE(PCH_PP_CONTROL, dev_priv->regfile.savePP_CONTROL);
                 I915_WRITE(RSTDBYCTL,
                            dev_priv->regfile.saveMCHBAR_RENDER_STANDBY);
+        } else if (IS_VALLEYVIEW(dev)) {
+                I915_WRITE(VLV_BLC_HIST_CTL(PIPE_A),
+                           dev_priv->regfile.saveBLC_HIST_CTL);
+                I915_WRITE(VLV_BLC_HIST_CTL(PIPE_B),
+                           dev_priv->regfile.saveBLC_HIST_CTL);
         } else {
                 I915_WRITE(PFIT_PGM_RATIOS, dev_priv->regfile.savePFIT_PGM_RATIOS);
-                I915_WRITE(BLC_PWM_CTL, dev_priv->regfile.saveBLC_PWM_CTL);
                 I915_WRITE(BLC_HIST_CTL, dev_priv->regfile.saveBLC_HIST_CTL);
                 I915_WRITE(PP_ON_DELAYS, dev_priv->regfile.savePP_ON_DELAYS);
                 I915_WRITE(PP_OFF_DELAYS, dev_priv->regfile.savePP_OFF_DELAYS);
@@ -312,8 +298,6 @@ static void i915_restore_display(struct drm_device *dev)
                 I915_WRITE(PP_CONTROL, dev_priv->regfile.savePP_CONTROL);
         }
 
-        spin_unlock_irqrestore(&dev_priv->backlight.lock, flags);
-
         /* only restore FBC info on the platform that supports FBC*/
         intel_disable_fbc(dev);
         if (I915_HAS_FBC(dev)) {
diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
index fdce8824723c..05d8b1680c22 100644
--- a/drivers/gpu/drm/i915/i915_sysfs.c
+++ b/drivers/gpu/drm/i915/i915_sysfs.c
@@ -32,6 +32,8 @@
 #include "intel_drv.h"
 #include "i915_drv.h"
 
+#define dev_to_drm_minor(d) dev_get_drvdata((d))
+
 #ifdef CONFIG_PM
 static u32 calc_residency(struct drm_device *dev, const u32 reg)
 {
@@ -66,14 +68,14 @@ static u32 calc_residency(struct drm_device *dev, const u32 reg)
 static ssize_t
 show_rc6_mask(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-        struct drm_minor *dminor = container_of(kdev, struct drm_minor, kdev);
+        struct drm_minor *dminor = dev_to_drm_minor(kdev);
         return snprintf(buf, PAGE_SIZE, "%x\n", intel_enable_rc6(dminor->dev));
 }
 
 static ssize_t
 show_rc6_ms(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-        struct drm_minor *dminor = container_of(kdev, struct drm_minor, kdev);
+        struct drm_minor *dminor = dev_get_drvdata(kdev);
         u32 rc6_residency = calc_residency(dminor->dev, GEN6_GT_GFX_RC6);
         return snprintf(buf, PAGE_SIZE, "%u\n", rc6_residency);
 }
@@ -81,7 +83,7 @@ show_rc6_ms(struct device *kdev, struct device_attribute *attr, char *buf)
 static ssize_t
 show_rc6p_ms(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-        struct drm_minor *dminor = container_of(kdev, struct drm_minor, kdev);
+        struct drm_minor *dminor = dev_to_drm_minor(kdev);
         u32 rc6p_residency = calc_residency(dminor->dev, GEN6_GT_GFX_RC6p);
         if (IS_VALLEYVIEW(dminor->dev))
                 rc6p_residency = 0;
@@ -91,7 +93,7 @@ show_rc6p_ms(struct device *kdev, struct device_attribute *attr, char *buf)
 static ssize_t
 show_rc6pp_ms(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-        struct drm_minor *dminor = container_of(kdev, struct drm_minor, kdev);
+        struct drm_minor *dminor = dev_to_drm_minor(kdev);
         u32 rc6pp_residency = calc_residency(dminor->dev, GEN6_GT_GFX_RC6pp);
         if (IS_VALLEYVIEW(dminor->dev))
                 rc6pp_residency = 0;
@@ -137,7 +139,7 @@ i915_l3_read(struct file *filp, struct kobject *kobj,
              loff_t offset, size_t count)
 {
         struct device *dev = container_of(kobj, struct device, kobj);
-        struct drm_minor *dminor = container_of(dev, struct drm_minor, kdev);
+        struct drm_minor *dminor = dev_to_drm_minor(dev);
         struct drm_device *drm_dev = dminor->dev;
         struct drm_i915_private *dev_priv = drm_dev->dev_private;
         int slice = (int)(uintptr_t)attr->private;
@@ -173,7 +175,7 @@ i915_l3_write(struct file *filp, struct kobject *kobj,
               loff_t offset, size_t count)
 {
         struct device *dev = container_of(kobj, struct device, kobj);
-        struct drm_minor *dminor = container_of(dev, struct drm_minor, kdev);
+        struct drm_minor *dminor = dev_to_drm_minor(dev);
         struct drm_device *drm_dev = dminor->dev;
         struct drm_i915_private *dev_priv = drm_dev->dev_private;
         struct i915_hw_context *ctx;
@@ -246,7 +248,7 @@ static struct bin_attribute dpf_attrs_1 = {
 static ssize_t gt_cur_freq_mhz_show(struct device *kdev,
                                     struct device_attribute *attr, char *buf)
 {
-        struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+        struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
@@ -269,7 +271,7 @@ static ssize_t gt_cur_freq_mhz_show(struct device *kdev,
 static ssize_t vlv_rpe_freq_mhz_show(struct device *kdev,
                                      struct device_attribute *attr, char *buf)
 {
-        struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+        struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
 
@@ -279,7 +281,7 @@ static ssize_t vlv_rpe_freq_mhz_show(struct device *kdev,
 
 static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-        struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+        struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
@@ -300,7 +302,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
                                      struct device_attribute *attr,
                                      const char *buf, size_t count)
 {
-        struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+        struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 val, rp_state_cap, hw_max, hw_min, non_oc_max;
@@ -355,7 +357,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
 
 static ssize_t gt_min_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-        struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+        struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
@@ -376,7 +378,7 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev,
                                      struct device_attribute *attr,
                                      const char *buf, size_t count)
 {
-        struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+        struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 val, rp_state_cap, hw_max, hw_min;
@@ -437,7 +439,7 @@ static DEVICE_ATTR(gt_RPn_freq_mhz, S_IRUGO, gt_rp_mhz_show, NULL);
 /* For now we have a static number of RP states */
 static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-        struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+        struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 val, rp_state_cap;
@@ -485,7 +487,7 @@ static ssize_t error_state_read(struct file *filp, struct kobject *kobj,
 {
 
         struct device *kdev = container_of(kobj, struct device, kobj);
-        struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+        struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
         struct i915_error_state_file_priv error_priv;
         struct drm_i915_error_state_buf error_str;
@@ -520,7 +522,7 @@ static ssize_t error_state_write(struct file *file, struct kobject *kobj,
                                  loff_t off, size_t count)
 {
         struct device *kdev = container_of(kobj, struct device, kobj);
-        struct drm_minor *minor = container_of(kdev, struct drm_minor, kdev);
+        struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
         int ret;
 
@@ -550,19 +552,19 @@ void i915_setup_sysfs(struct drm_device *dev)
 
 #ifdef CONFIG_PM
         if (INTEL_INFO(dev)->gen >= 6) {
-                ret = sysfs_merge_group(&dev->primary->kdev.kobj,
+                ret = sysfs_merge_group(&dev->primary->kdev->kobj,
                                         &rc6_attr_group);
                 if (ret)
                         DRM_ERROR("RC6 residency sysfs setup failed\n");
         }
 #endif
         if (HAS_L3_DPF(dev)) {
-                ret = device_create_bin_file(&dev->primary->kdev, &dpf_attrs);
+                ret = device_create_bin_file(dev->primary->kdev, &dpf_attrs);
                 if (ret)
                         DRM_ERROR("l3 parity sysfs setup failed\n");
 
                 if (NUM_L3_SLICES(dev) > 1) {
-                        ret = device_create_bin_file(&dev->primary->kdev,
+                        ret = device_create_bin_file(dev->primary->kdev,
                                                      &dpf_attrs_1);
                         if (ret)
                                 DRM_ERROR("l3 parity slice 1 setup failed\n");
@@ -571,13 +573,13 @@ void i915_setup_sysfs(struct drm_device *dev)
 
         ret = 0;
         if (IS_VALLEYVIEW(dev))
-                ret = sysfs_create_files(&dev->primary->kdev.kobj, vlv_attrs);
+                ret = sysfs_create_files(&dev->primary->kdev->kobj, vlv_attrs);
         else if (INTEL_INFO(dev)->gen >= 6)
-                ret = sysfs_create_files(&dev->primary->kdev.kobj, gen6_attrs);
+                ret = sysfs_create_files(&dev->primary->kdev->kobj, gen6_attrs);
         if (ret)
                 DRM_ERROR("RPS sysfs setup failed\n");
 
-        ret = sysfs_create_bin_file(&dev->primary->kdev.kobj,
+        ret = sysfs_create_bin_file(&dev->primary->kdev->kobj,
                                     &error_state_attr);
         if (ret)
                 DRM_ERROR("error_state sysfs setup failed\n");
@@ -585,14 +587,14 @@ void i915_setup_sysfs(struct drm_device *dev)
 
 void i915_teardown_sysfs(struct drm_device *dev)
 {
-        sysfs_remove_bin_file(&dev->primary->kdev.kobj, &error_state_attr);
+        sysfs_remove_bin_file(&dev->primary->kdev->kobj, &error_state_attr);
         if (IS_VALLEYVIEW(dev))
-                sysfs_remove_files(&dev->primary->kdev.kobj, vlv_attrs);
+                sysfs_remove_files(&dev->primary->kdev->kobj, vlv_attrs);
         else
-                sysfs_remove_files(&dev->primary->kdev.kobj, gen6_attrs);
-        device_remove_bin_file(&dev->primary->kdev,  &dpf_attrs_1);
-        device_remove_bin_file(&dev->primary->kdev,  &dpf_attrs);
+                sysfs_remove_files(&dev->primary->kdev->kobj, gen6_attrs);
+        device_remove_bin_file(dev->primary->kdev,  &dpf_attrs_1);
+        device_remove_bin_file(dev->primary->kdev,  &dpf_attrs);
 #ifdef CONFIG_PM
-        sysfs_unmerge_group(&dev->primary->kdev.kobj, &rc6_attr_group);
+        sysfs_unmerge_group(&dev->primary->kdev->kobj, &rc6_attr_group);
 #endif
 }
diff --git a/drivers/gpu/drm/i915/i915_ums.c b/drivers/gpu/drm/i915/i915_ums.c
index 967da4772c44..caa18e855815 100644
--- a/drivers/gpu/drm/i915/i915_ums.c
+++ b/drivers/gpu/drm/i915/i915_ums.c
@@ -270,6 +270,18 @@ void i915_save_display_reg(struct drm_device *dev)
         }
         /* FIXME: regfile.save TV & SDVO state */
 
+        /* Backlight */
+        if (HAS_PCH_SPLIT(dev)) {
+                dev_priv->regfile.saveBLC_PWM_CTL = I915_READ(BLC_PWM_PCH_CTL1);
+                dev_priv->regfile.saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_PCH_CTL2);
+                dev_priv->regfile.saveBLC_CPU_PWM_CTL = I915_READ(BLC_PWM_CPU_CTL);
+                dev_priv->regfile.saveBLC_CPU_PWM_CTL2 = I915_READ(BLC_PWM_CPU_CTL2);
+        } else {
+                dev_priv->regfile.saveBLC_PWM_CTL = I915_READ(BLC_PWM_CTL);
+                if (INTEL_INFO(dev)->gen >= 4)
+                        dev_priv->regfile.saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_CTL2);
+        }
+
         return;
 }
 
@@ -280,6 +292,21 @@ void i915_restore_display_reg(struct drm_device *dev)
         int dpll_b_reg, fpb0_reg, fpb1_reg;
         int i;
 
+        /* Backlight */
+        if (HAS_PCH_SPLIT(dev)) {
+                I915_WRITE(BLC_PWM_PCH_CTL1, dev_priv->regfile.saveBLC_PWM_CTL);
+                I915_WRITE(BLC_PWM_PCH_CTL2, dev_priv->regfile.saveBLC_PWM_CTL2);
+                /* NOTE: BLC_PWM_CPU_CTL must be written after BLC_PWM_CPU_CTL2;
+                 * otherwise we get blank eDP screen after S3 on some machines
+                 */
+                I915_WRITE(BLC_PWM_CPU_CTL2, dev_priv->regfile.saveBLC_CPU_PWM_CTL2);
+                I915_WRITE(BLC_PWM_CPU_CTL, dev_priv->regfile.saveBLC_CPU_PWM_CTL);
+        } else {
+                if (INTEL_INFO(dev)->gen >= 4)
+                        I915_WRITE(BLC_PWM_CTL2, dev_priv->regfile.saveBLC_PWM_CTL2);
+                I915_WRITE(BLC_PWM_CTL, dev_priv->regfile.saveBLC_PWM_CTL);
+        }
+
         /* Display port ratios (must be done before clock is set) */
         if (SUPPORTS_INTEGRATED_DP(dev)) {
                 I915_WRITE(_PIPEA_DATA_M_G4X, dev_priv->regfile.savePIPEA_GMCH_DATA_M);
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index e29bcae1ef81..6dd622d733b9 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -624,11 +624,11 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
 
         aux_channel = child->raw[25];
 
-        is_dvi = child->common.device_type & (1 << 4);
-        is_dp = child->common.device_type & (1 << 2);
-        is_crt = child->common.device_type & (1 << 0);
-        is_hdmi = is_dvi && (child->common.device_type & (1 << 11)) == 0;
-        is_edp = is_dp && (child->common.device_type & (1 << 12));
+        is_dvi = child->common.device_type & DEVICE_TYPE_TMDS_DVI_SIGNALING;
+        is_dp = child->common.device_type & DEVICE_TYPE_DISPLAYPORT_OUTPUT;
+        is_crt = child->common.device_type & DEVICE_TYPE_ANALOG_OUTPUT;
+        is_hdmi = is_dvi && (child->common.device_type & DEVICE_TYPE_NOT_HDMI_OUTPUT) == 0;
+        is_edp = is_dp && (child->common.device_type & DEVICE_TYPE_INTERNAL_CONNECTOR);
 
         info->supports_dvi = is_dvi;
         info->supports_hdmi = is_hdmi;
diff --git a/drivers/gpu/drm/i915/intel_bios.h b/drivers/gpu/drm/i915/intel_bios.h
index 287cc5a21c2e..f580a2b0ddd3 100644
--- a/drivers/gpu/drm/i915/intel_bios.h
+++ b/drivers/gpu/drm/i915/intel_bios.h
@@ -638,6 +638,40 @@ int intel_parse_bios(struct drm_device *dev);
 #define  DEVICE_TYPE_DP         0x68C6
 #define  DEVICE_TYPE_eDP        0x78C6
 
+#define  DEVICE_TYPE_CLASS_EXTENSION    (1 << 15)
+#define  DEVICE_TYPE_POWER_MANAGEMENT   (1 << 14)
+#define  DEVICE_TYPE_HOTPLUG_SIGNALING  (1 << 13)
+#define  DEVICE_TYPE_INTERNAL_CONNECTOR (1 << 12)
+#define  DEVICE_TYPE_NOT_HDMI_OUTPUT    (1 << 11)
+#define  DEVICE_TYPE_MIPI_OUTPUT        (1 << 10)
+#define  DEVICE_TYPE_COMPOSITE_OUTPUT   (1 << 9)
+#define  DEVICE_TYPE_DUAL_CHANNEL       (1 << 8)
+#define  DEVICE_TYPE_HIGH_SPEED_LINK    (1 << 6)
+#define  DEVICE_TYPE_LVDS_SINGALING     (1 << 5)
+#define  DEVICE_TYPE_TMDS_DVI_SIGNALING (1 << 4)
+#define  DEVICE_TYPE_VIDEO_SIGNALING    (1 << 3)
+#define  DEVICE_TYPE_DISPLAYPORT_OUTPUT (1 << 2)
+#define  DEVICE_TYPE_DIGITAL_OUTPUT     (1 << 1)
+#define  DEVICE_TYPE_ANALOG_OUTPUT      (1 << 0)
+
+/*
+ * Bits we care about when checking for DEVICE_TYPE_eDP
+ * Depending on the system, the other bits may or may not
+ * be set for eDP outputs.
+ */
+#define DEVICE_TYPE_eDP_BITS \
+        (DEVICE_TYPE_INTERNAL_CONNECTOR | \
+         DEVICE_TYPE_NOT_HDMI_OUTPUT | \
+         DEVICE_TYPE_MIPI_OUTPUT | \
+         DEVICE_TYPE_COMPOSITE_OUTPUT | \
+         DEVICE_TYPE_DUAL_CHANNEL | \
+         DEVICE_TYPE_LVDS_SINGALING | \
+         DEVICE_TYPE_TMDS_DVI_SIGNALING | \
+         DEVICE_TYPE_VIDEO_SIGNALING | \
+         DEVICE_TYPE_DISPLAYPORT_OUTPUT | \
+         DEVICE_TYPE_DIGITAL_OUTPUT | \
+         DEVICE_TYPE_ANALOG_OUTPUT)
+
 /* define the DVO port for HDMI output type */
 #define         DVO_B           1
 #define         DVO_C           2
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 2e01bd3a5d8c..b5b1b9b23adf 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -822,16 +822,15 @@ void intel_crt_init(struct drm_device *dev)
         crt->base.mode_set = intel_crt_mode_set;
         crt->base.disable = intel_disable_crt;
         crt->base.enable = intel_enable_crt;
-        if (IS_HASWELL(dev))
-                crt->base.get_config = hsw_crt_get_config;
-        else
-                crt->base.get_config = intel_crt_get_config;
         if (I915_HAS_HOTPLUG(dev))
                 crt->base.hpd_pin = HPD_CRT;
-        if (HAS_DDI(dev))
+        if (HAS_DDI(dev)) {
+                crt->base.get_config = hsw_crt_get_config;
                 crt->base.get_hw_state = intel_ddi_get_hw_state;
-        else
+        } else {
+                crt->base.get_config = intel_crt_get_config;
                 crt->base.get_hw_state = intel_crt_get_hw_state;
+        }
         intel_connector->get_hw_state = intel_connector_get_hw_state;
 
         drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs);
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 31f4fe271388..1591576a6101 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -72,6 +72,45 @@ static const u32 hsw_ddi_translations_hdmi[] = {
         0x80FFFFFF, 0x00030002, /* 11:  1000            1000            0   */
 };
 
+static const u32 bdw_ddi_translations_edp[] = {
+        0x00FFFFFF, 0x00000012,         /* DP parameters */
+        0x00EBAFFF, 0x00020011,
+        0x00C71FFF, 0x0006000F,
+        0x00FFFFFF, 0x00020011,
+        0x00DB6FFF, 0x0005000F,
+        0x00BEEFFF, 0x000A000C,
+        0x00FFFFFF, 0x0005000F,
+        0x00DB6FFF, 0x000A000C,
+        0x00FFFFFF, 0x000A000C,
+        0x00FFFFFF, 0x00140006          /* HDMI parameters 800mV 0dB*/
+};
+
+static const u32 bdw_ddi_translations_dp[] = {
+        0x00FFFFFF, 0x0007000E,         /* DP parameters */
+        0x00D75FFF, 0x000E000A,
+        0x00BEFFFF, 0x00140006,
+        0x00FFFFFF, 0x000E000A,
+        0x00D75FFF, 0x00180004,
+        0x80CB2FFF, 0x001B0002,
+        0x00F7DFFF, 0x00180004,
+        0x80D75FFF, 0x001B0002,
+        0x80FFFFFF, 0x001B0002,
+        0x00FFFFFF, 0x00140006          /* HDMI parameters 800mV 0dB*/
+};
+
+static const u32 bdw_ddi_translations_fdi[] = {
+        0x00FFFFFF, 0x0001000E,         /* FDI parameters */
+        0x00D75FFF, 0x0004000A,
+        0x00C30FFF, 0x00070006,
+        0x00AAAFFF, 0x000C0000,
+        0x00FFFFFF, 0x0004000A,
+        0x00D75FFF, 0x00090004,
+        0x00C30FFF, 0x000C0000,
+        0x00FFFFFF, 0x00070006,
+        0x00D75FFF, 0x000C0000,
+        0x00FFFFFF, 0x00140006          /* HDMI parameters 800mV 0dB*/
+};
+
 enum port intel_ddi_get_encoder_port(struct intel_encoder *intel_encoder)
 {
         struct drm_encoder *encoder = &intel_encoder->base;
@@ -92,8 +131,9 @@ enum port intel_ddi_get_encoder_port(struct intel_encoder *intel_encoder)
         }
 }
 
-/* On Haswell, DDI port buffers must be programmed with correct values
- * in advance. The buffer values are different for FDI and DP modes,
+/*
+ * Starting with Haswell, DDI port buffers must be programmed with correct
+ * values in advance. The buffer values are different for FDI and DP modes,
  * but the HDMI/DVI fields are shared among those. So we program the DDI
  * in either FDI or DP modes only, as HDMI connections will work with both
  * of those
@@ -103,10 +143,47 @@ static void intel_prepare_ddi_buffers(struct drm_device *dev, enum port port)
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 reg;
         int i;
-        const u32 *ddi_translations = (port == PORT_E) ?
-                hsw_ddi_translations_fdi :
-                hsw_ddi_translations_dp;
         int hdmi_level = dev_priv->vbt.ddi_port_info[port].hdmi_level_shift;
+        const u32 *ddi_translations_fdi;
+        const u32 *ddi_translations_dp;
+        const u32 *ddi_translations_edp;
+        const u32 *ddi_translations;
+
+        if (IS_BROADWELL(dev)) {
+                ddi_translations_fdi = bdw_ddi_translations_fdi;
+                ddi_translations_dp = bdw_ddi_translations_dp;
+                ddi_translations_edp = bdw_ddi_translations_edp;
+        } else if (IS_HASWELL(dev)) {
+                ddi_translations_fdi = hsw_ddi_translations_fdi;
+                ddi_translations_dp = hsw_ddi_translations_dp;
+                ddi_translations_edp = hsw_ddi_translations_dp;
+        } else {
+                WARN(1, "ddi translation table missing\n");
+                ddi_translations_edp = bdw_ddi_translations_dp;
+                ddi_translations_fdi = bdw_ddi_translations_fdi;
+                ddi_translations_dp = bdw_ddi_translations_dp;
+        }
+
+        switch (port) {
+        case PORT_A:
+                ddi_translations = ddi_translations_edp;
+                break;
+        case PORT_B:
+        case PORT_C:
+                ddi_translations = ddi_translations_dp;
+                break;
+        case PORT_D:
+                if (intel_dpd_is_edp(dev))
+                        ddi_translations = ddi_translations_edp;
+                else
+                        ddi_translations = ddi_translations_dp;
+                break;
+        case PORT_E:
+                ddi_translations = ddi_translations_fdi;
+                break;
+        default:
+                BUG();
+        }
 
         for (i = 0, reg = DDI_BUF_TRANS(port);
              i < ARRAY_SIZE(hsw_ddi_translations_fdi); i++) {
@@ -756,7 +833,8 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc)
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc);
         struct drm_encoder *encoder = &intel_encoder->base;
-        struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         enum pipe pipe = intel_crtc->pipe;
         enum transcoder cpu_transcoder = intel_crtc->config.cpu_transcoder;
         enum port port = intel_ddi_get_encoder_port(intel_encoder);
@@ -792,10 +870,11 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc)
         if (cpu_transcoder == TRANSCODER_EDP) {
                 switch (pipe) {
                 case PIPE_A:
-                        /* Can only use the always-on power well for eDP when
-                         * not using the panel fitter, and when not using motion
-                          * blur mitigation (which we don't support). */
-                        if (intel_crtc->config.pch_pfit.enabled)
+                        /* On Haswell, can only use the always-on power well for
+                         * eDP when not using the panel fitter, and when not
+                         * using motion blur mitigation (which we don't
+                         * support). */
+                        if (IS_HASWELL(dev) && intel_crtc->config.pch_pfit.enabled)
                                 temp |= TRANS_DDI_EDP_INPUT_A_ONOFF;
                         else
                                 temp |= TRANS_DDI_EDP_INPUT_A_ON;
@@ -1156,18 +1235,29 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder)
 
 int intel_ddi_get_cdclk_freq(struct drm_i915_private *dev_priv)
 {
+        struct drm_device *dev = dev_priv->dev;
         uint32_t lcpll = I915_READ(LCPLL_CTL);
+        uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
 
-        if (lcpll & LCPLL_CD_SOURCE_FCLK)
+        if (lcpll & LCPLL_CD_SOURCE_FCLK) {
                 return 800000;
-        else if (I915_READ(HSW_FUSE_STRAP) & HSW_CDCLK_LIMIT)
+        } else if (I915_READ(HSW_FUSE_STRAP) & HSW_CDCLK_LIMIT) {
                 return 450000;
-        else if ((lcpll & LCPLL_CLK_FREQ_MASK) == LCPLL_CLK_FREQ_450)
+        } else if (freq == LCPLL_CLK_FREQ_450) {
                 return 450000;
-        else if (IS_ULT(dev_priv->dev))
-                return 337500;
-        else
-                return 540000;
+        } else if (IS_HASWELL(dev)) {
+                if (IS_ULT(dev))
+                        return 337500;
+                else
+                        return 540000;
+        } else {
+                if (freq == LCPLL_CLK_FREQ_54O_BDW)
+                        return 540000;
+                else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
+                        return 337500;
+                else
+                        return 675000;
+        }
 }
 
 void intel_ddi_pll_init(struct drm_device *dev)
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 752d83019f36..3b7f1c4eb48e 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -2164,7 +2164,7 @@ static int ironlake_update_plane(struct drm_crtc *crtc,
         else
                 dspcntr &= ~DISPPLANE_TILED;
 
-        if (IS_HASWELL(dev))
+        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
                 dspcntr &= ~DISPPLANE_TRICKLE_FEED_DISABLE;
         else
                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
@@ -2184,7 +2184,7 @@ static int ironlake_update_plane(struct drm_crtc *crtc,
         I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
         I915_MODIFY_DISPBASE(DSPSURF(plane),
                              i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset);
-        if (IS_HASWELL(dev)) {
+        if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
                 I915_WRITE(DSPOFFSET(plane), (y << 16) | x);
         } else {
                 I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
@@ -3401,15 +3401,26 @@ void hsw_enable_ips(struct intel_crtc *crtc)
          * only after intel_enable_plane. And intel_enable_plane already waits
          * for a vblank, so all we need to do here is to enable the IPS bit. */
         assert_plane_enabled(dev_priv, crtc->plane);
-        I915_WRITE(IPS_CTL, IPS_ENABLE);
-
-        /* The bit only becomes 1 in the next vblank, so this wait here is
-         * essentially intel_wait_for_vblank. If we don't have this and don't
-         * wait for vblanks until the end of crtc_enable, then the HW state
-         * readout code will complain that the expected IPS_CTL value is not the
-         * one we read. */
-        if (wait_for(I915_READ_NOTRACE(IPS_CTL) & IPS_ENABLE, 50))
-                DRM_ERROR("Timed out waiting for IPS enable\n");
+        if (IS_BROADWELL(crtc->base.dev)) {
+                mutex_lock(&dev_priv->rps.hw_lock);
+                WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0xc0000000));
+                mutex_unlock(&dev_priv->rps.hw_lock);
+                /* Quoting Art Runyan: "its not safe to expect any particular
+                 * value in IPS_CTL bit 31 after enabling IPS through the
+                 * mailbox." Therefore we need to defer waiting on the state
+                 * change.
+                 * TODO: need to fix this for state checker
+                 */
+        } else {
+                I915_WRITE(IPS_CTL, IPS_ENABLE);
+                /* The bit only becomes 1 in the next vblank, so this wait here
+                 * is essentially intel_wait_for_vblank. If we don't have this
+                 * and don't wait for vblanks until the end of crtc_enable, then
+                 * the HW state readout code will complain that the expected
+                 * IPS_CTL value is not the one we read. */
+                if (wait_for(I915_READ_NOTRACE(IPS_CTL) & IPS_ENABLE, 50))
+                        DRM_ERROR("Timed out waiting for IPS enable\n");
+        }
 }
 
 void hsw_disable_ips(struct intel_crtc *crtc)
@@ -3421,7 +3432,12 @@ void hsw_disable_ips(struct intel_crtc *crtc)
                 return;
 
         assert_plane_enabled(dev_priv, crtc->plane);
-        I915_WRITE(IPS_CTL, 0);
+        if (IS_BROADWELL(crtc->base.dev)) {
+                mutex_lock(&dev_priv->rps.hw_lock);
+                WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
+                mutex_unlock(&dev_priv->rps.hw_lock);
+        } else
+                I915_WRITE(IPS_CTL, 0);
         POSTING_READ(IPS_CTL);
 
         /* We need to wait for a vblank before we can disable the plane. */
@@ -4420,7 +4436,7 @@ static bool ironlake_check_fdi_lanes(struct drm_device *dev, enum pipe pipe,
                 return false;
         }
 
-        if (IS_HASWELL(dev)) {
+        if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
                 if (pipe_config->fdi_lanes > 2) {
                         DRM_DEBUG_KMS("only 2 lanes on haswell, required: %i lanes\n",
                                       pipe_config->fdi_lanes);
@@ -5994,14 +6010,16 @@ static void intel_set_pipe_csc(struct drm_crtc *crtc)
 
 static void haswell_set_pipeconf(struct drm_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        enum pipe pipe = intel_crtc->pipe;
         enum transcoder cpu_transcoder = intel_crtc->config.cpu_transcoder;
         uint32_t val;
 
         val = 0;
 
-        if (intel_crtc->config.dither)
+        if (IS_HASWELL(dev) && intel_crtc->config.dither)
                 val |= (PIPECONF_DITHER_EN | PIPECONF_DITHER_TYPE_SP);
 
         if (intel_crtc->config.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
@@ -6014,6 +6032,33 @@ static void haswell_set_pipeconf(struct drm_crtc *crtc)
 
         I915_WRITE(GAMMA_MODE(intel_crtc->pipe), GAMMA_MODE_MODE_8BIT);
         POSTING_READ(GAMMA_MODE(intel_crtc->pipe));
+
+        if (IS_BROADWELL(dev)) {
+                val = 0;
+
+                switch (intel_crtc->config.pipe_bpp) {
+                case 18:
+                        val |= PIPEMISC_DITHER_6_BPC;
+                        break;
+                case 24:
+                        val |= PIPEMISC_DITHER_8_BPC;
+                        break;
+                case 30:
+                        val |= PIPEMISC_DITHER_10_BPC;
+                        break;
+                case 36:
+                        val |= PIPEMISC_DITHER_12_BPC;
+                        break;
+                default:
+                        /* Case prevented by pipe_config_set_bpp. */
+                        BUG();
+                }
+
+                if (intel_crtc->config.dither)
+                        val |= PIPEMISC_DITHER_ENABLE | PIPEMISC_DITHER_TYPE_SP;
+
+                I915_WRITE(PIPEMISC(pipe), val);
+        }
 }
 
 static bool ironlake_compute_clocks(struct drm_crtc *crtc,
@@ -7165,6 +7210,11 @@ static void ironlake_write_eld(struct drm_connector *connector,
                 aud_config = IBX_AUD_CFG(pipe);
                 aud_cntl_st = IBX_AUD_CNTL_ST(pipe);
                 aud_cntrl_st2 = IBX_AUD_CNTL_ST2;
+        } else if (IS_VALLEYVIEW(connector->dev)) {
+                hdmiw_hdmiedid = VLV_HDMIW_HDMIEDID(pipe);
+                aud_config = VLV_AUD_CFG(pipe);
+                aud_cntl_st = VLV_AUD_CNTL_ST(pipe);
+                aud_cntrl_st2 = VLV_AUD_CNTL_ST2;
         } else {
                 hdmiw_hdmiedid = CPT_HDMIW_HDMIEDID(pipe);
                 aud_config = CPT_AUD_CFG(pipe);
@@ -7174,8 +7224,19 @@ static void ironlake_write_eld(struct drm_connector *connector,
 
         DRM_DEBUG_DRIVER("ELD on pipe %c\n", pipe_name(pipe));
 
-        i = I915_READ(aud_cntl_st);
-        i = (i >> 29) & DIP_PORT_SEL_MASK;              /* DIP_Port_Select, 0x1 = PortB */
+        if (IS_VALLEYVIEW(connector->dev))  {
+                struct intel_encoder *intel_encoder;
+                struct intel_digital_port *intel_dig_port;
+
+                intel_encoder = intel_attached_encoder(connector);
+                intel_dig_port = enc_to_dig_port(&intel_encoder->base);
+                i = intel_dig_port->port;
+        } else {
+                i = I915_READ(aud_cntl_st);
+                i = (i >> 29) & DIP_PORT_SEL_MASK;
+                /* DIP_Port_Select, 0x1 = PortB */
+        }
+
         if (!i) {
                 DRM_DEBUG_DRIVER("Audio directed to unknown port\n");
                 /* operate blindly on all ports */
@@ -7319,7 +7380,7 @@ static void ivb_update_cursor(struct drm_crtc *crtc, u32 base)
                         cntl &= ~(CURSOR_MODE | MCURSOR_GAMMA_ENABLE);
                         cntl |= CURSOR_MODE_DISABLE;
                 }
-                if (IS_HASWELL(dev)) {
+                if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
                         cntl |= CURSOR_PIPE_CSC_ENABLE;
                         cntl &= ~CURSOR_TRICKLE_FEED_DISABLE;
                 }
@@ -7375,7 +7436,7 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
         if (!visible && !intel_crtc->cursor_visible)
                 return;
 
-        if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
+        if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_BROADWELL(dev)) {
                 I915_WRITE(CURPOS_IVB(pipe), pos);
                 ivb_update_cursor(crtc, base);
         } else {
@@ -10049,6 +10110,18 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
 }
 
+enum pipe intel_get_pipe_from_connector(struct intel_connector *connector)
+{
+        struct drm_encoder *encoder = connector->base.encoder;
+
+        WARN_ON(!mutex_is_locked(&connector->base.dev->mode_config.mutex));
+
+        if (!encoder)
+                return INVALID_PIPE;
+
+        return to_intel_crtc(encoder->crtc)->pipe;
+}
+
 int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
                                 struct drm_file *file)
 {
@@ -10064,7 +10137,7 @@ int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
 
         if (!drmmode_obj) {
                 DRM_ERROR("no such CRTC id\n");
-                return -EINVAL;
+                return -ENOENT;
         }
 
         crtc = to_intel_crtc(obj_to_crtc(drmmode_obj));
@@ -10500,7 +10573,7 @@ static void intel_init_display(struct drm_device *dev)
                         dev_priv->display.write_eld = ironlake_write_eld;
                         dev_priv->display.modeset_global_resources =
                                 ivb_modeset_global_resources;
-                } else if (IS_HASWELL(dev)) {
+                } else if (IS_HASWELL(dev) || IS_GEN8(dev)) {
                         dev_priv->display.fdi_link_train = hsw_fdi_link_train;
                         dev_priv->display.write_eld = haswell_write_eld;
                         dev_priv->display.modeset_global_resources =
@@ -10511,6 +10584,7 @@ static void intel_init_display(struct drm_device *dev)
         } else if (IS_VALLEYVIEW(dev)) {
                 dev_priv->display.modeset_global_resources =
                         valleyview_modeset_global_resources;
+                dev_priv->display.write_eld = ironlake_write_eld;
         }
 
         /* Default just returns -ENODEV to indicate unsupported */
@@ -10534,9 +10608,12 @@ static void intel_init_display(struct drm_device *dev)
                 dev_priv->display.queue_flip = intel_gen6_queue_flip;
                 break;
         case 7:
+        case 8: /* FIXME(BDW): Check that the gen8 RCS flip works. */
                 dev_priv->display.queue_flip = intel_gen7_queue_flip;
                 break;
         }
+
+        intel_panel_init_backlight_funcs(dev);
 }
 
 /*
@@ -10573,17 +10650,6 @@ static void quirk_invert_brightness(struct drm_device *dev)
         DRM_INFO("applying inverted panel brightness quirk\n");
 }
 
-/*
- * Some machines (Dell XPS13) suffer broken backlight controls if
- * BLM_PCH_PWM_ENABLE is set.
- */
-static void quirk_no_pcm_pwm_enable(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        dev_priv->quirks |= QUIRK_NO_PCH_PWM_ENABLE;
-        DRM_INFO("applying no-PCH_PWM_ENABLE quirk\n");
-}
-
 struct intel_quirk {
         int device;
         int subsystem_vendor;
@@ -10643,11 +10709,6 @@ static struct intel_quirk intel_quirks[] = {
          * seem to use inverted backlight PWM.
          */
         { 0x2a42, 0x1025, PCI_ANY_ID, quirk_invert_brightness },
-
-        /* Dell XPS13 HD Sandy Bridge */
-        { 0x0116, 0x1028, 0x052e, quirk_no_pcm_pwm_enable },
-        /* Dell XPS13 HD and XPS13 FHD Ivy Bridge */
-        { 0x0166, 0x1028, 0x058b, quirk_no_pcm_pwm_enable },
 };
 
 static void intel_init_quirks(struct drm_device *dev)
@@ -11189,12 +11250,11 @@ void intel_modeset_cleanup(struct drm_device *dev)
         /* flush any delayed tasks or pending work */
         flush_scheduled_work();
 
-        /* destroy backlight, if any, before the connectors */
-        intel_panel_destroy_backlight(dev);
-
-        /* destroy the sysfs files before encoders/connectors */
-        list_for_each_entry(connector, &dev->mode_config.connector_list, head)
+        /* destroy the backlight and sysfs files before encoders/connectors */
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                intel_panel_destroy_backlight(connector);
                 drm_sysfs_connector_remove(connector);
+        }
 
         drm_mode_config_cleanup(dev);
 
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 34d605762a60..dbe4840d6fb8 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -405,6 +405,7 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
         uint32_t status;
         int try, precharge, clock = 0;
         bool has_aux_irq = INTEL_INFO(dev)->gen >= 5 && !IS_VALLEYVIEW(dev);
+        uint32_t timeout;
 
         /* dp aux is extremely sensitive to irq latency, hence request the
          * lowest possible wakeup latency and so prevent the cpu from going into
@@ -419,6 +420,11 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
         else
                 precharge = 5;
 
+        if (IS_BROADWELL(dev) && ch_ctl == DPA_AUX_CH_CTL)
+                timeout = DP_AUX_CH_CTL_TIME_OUT_600us;
+        else
+                timeout = DP_AUX_CH_CTL_TIME_OUT_400us;
+
         intel_aux_display_runtime_get(dev_priv);
 
         /* Try to wait for any previous AUX channel activity */
@@ -454,7 +460,7 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
                         I915_WRITE(ch_ctl,
                                    DP_AUX_CH_CTL_SEND_BUSY |
                                    (has_aux_irq ? DP_AUX_CH_CTL_INTERRUPT : 0) |
-                                   DP_AUX_CH_CTL_TIME_OUT_400us |
+                                   timeout |
                                    (send_bytes << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) |
                                    (precharge << DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT) |
                                    (aux_clock_divider << DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT) |
@@ -747,7 +753,7 @@ intel_dp_i2c_init(struct intel_dp *intel_dp,
         strncpy(intel_dp->adapter.name, name, sizeof(intel_dp->adapter.name) - 1);
         intel_dp->adapter.name[sizeof(intel_dp->adapter.name) - 1] = '\0';
         intel_dp->adapter.algo_data = &intel_dp->algo;
-        intel_dp->adapter.dev.parent = &intel_connector->base.kdev;
+        intel_dp->adapter.dev.parent = intel_connector->base.kdev;
 
         ret = i2c_dp_aux_add_bus(&intel_dp->adapter);
         return ret;
@@ -1249,7 +1255,6 @@ void ironlake_edp_backlight_on(struct intel_dp *intel_dp)
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int pipe = to_intel_crtc(intel_dig_port->base.base.crtc)->pipe;
         u32 pp;
         u32 pp_ctrl_reg;
 
@@ -1272,7 +1277,7 @@ void ironlake_edp_backlight_on(struct intel_dp *intel_dp)
         I915_WRITE(pp_ctrl_reg, pp);
         POSTING_READ(pp_ctrl_reg);
 
-        intel_panel_enable_backlight(dev, pipe);
+        intel_panel_enable_backlight(intel_dp->attached_connector);
 }
 
 void ironlake_edp_backlight_off(struct intel_dp *intel_dp)
@@ -1285,7 +1290,7 @@ void ironlake_edp_backlight_off(struct intel_dp *intel_dp)
         if (!is_edp(intel_dp))
                 return;
 
-        intel_panel_disable_backlight(dev);
+        intel_panel_disable_backlight(intel_dp->attached_connector);
 
         DRM_DEBUG_KMS("\n");
         pp = ironlake_get_pp_control(intel_dp);
@@ -1611,6 +1616,7 @@ static void intel_edp_psr_enable_source(struct intel_dp *intel_dp)
         uint32_t max_sleep_time = 0x1f;
         uint32_t idle_frames = 1;
         uint32_t val = 0x0;
+        const uint32_t link_entry_time = EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES;
 
         if (intel_dp->psr_dpcd[1] & DP_PSR_NO_TRAIN_ON_EXIT) {
                 val |= EDP_PSR_LINK_STANDBY;
@@ -1621,7 +1627,7 @@ static void intel_edp_psr_enable_source(struct intel_dp *intel_dp)
                 val |= EDP_PSR_LINK_DISABLE;
 
         I915_WRITE(EDP_PSR_CTL(dev), val |
-                   EDP_PSR_MIN_LINK_ENTRY_TIME_8_LINES |
+                   IS_BROADWELL(dev) ? 0 : link_entry_time |
                    max_sleep_time << EDP_PSR_MAX_SLEEP_TIME_SHIFT |
                    idle_frames << EDP_PSR_IDLE_FRAME_SHIFT |
                    EDP_PSR_ENABLE);
@@ -1958,7 +1964,7 @@ intel_dp_voltage_max(struct intel_dp *intel_dp)
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         enum port port = dp_to_dig_port(intel_dp)->port;
 
-        if (IS_VALLEYVIEW(dev))
+        if (IS_VALLEYVIEW(dev) || IS_BROADWELL(dev))
                 return DP_TRAIN_VOLTAGE_SWING_1200;
         else if (IS_GEN7(dev) && port == PORT_A)
                 return DP_TRAIN_VOLTAGE_SWING_800;
@@ -1974,7 +1980,18 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         enum port port = dp_to_dig_port(intel_dp)->port;
 
-        if (HAS_DDI(dev)) {
+        if (IS_BROADWELL(dev)) {
+                switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
+                case DP_TRAIN_VOLTAGE_SWING_400:
+                case DP_TRAIN_VOLTAGE_SWING_600:
+                        return DP_TRAIN_PRE_EMPHASIS_6;
+                case DP_TRAIN_VOLTAGE_SWING_800:
+                        return DP_TRAIN_PRE_EMPHASIS_3_5;
+                case DP_TRAIN_VOLTAGE_SWING_1200:
+                default:
+                        return DP_TRAIN_PRE_EMPHASIS_0;
+                }
+        } else if (IS_HASWELL(dev)) {
                 switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
                 case DP_TRAIN_VOLTAGE_SWING_400:
                         return DP_TRAIN_PRE_EMPHASIS_9_5;
@@ -2286,6 +2303,41 @@ intel_hsw_signal_levels(uint8_t train_set)
         }
 }
 
+static uint32_t
+intel_bdw_signal_levels(uint8_t train_set)
+{
+        int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK |
+                                         DP_TRAIN_PRE_EMPHASIS_MASK);
+        switch (signal_levels) {
+        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_0:
+                return DDI_BUF_EMP_400MV_0DB_BDW;       /* Sel0 */
+        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_3_5:
+                return DDI_BUF_EMP_400MV_3_5DB_BDW;     /* Sel1 */
+        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_6:
+                return DDI_BUF_EMP_400MV_6DB_BDW;       /* Sel2 */
+
+        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_0:
+                return DDI_BUF_EMP_600MV_0DB_BDW;       /* Sel3 */
+        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_3_5:
+                return DDI_BUF_EMP_600MV_3_5DB_BDW;     /* Sel4 */
+        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_6:
+                return DDI_BUF_EMP_600MV_6DB_BDW;       /* Sel5 */
+
+        case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_0:
+                return DDI_BUF_EMP_800MV_0DB_BDW;       /* Sel6 */
+        case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_3_5:
+                return DDI_BUF_EMP_800MV_3_5DB_BDW;     /* Sel7 */
+
+        case DP_TRAIN_VOLTAGE_SWING_1200 | DP_TRAIN_PRE_EMPHASIS_0:
+                return DDI_BUF_EMP_1200MV_0DB_BDW;      /* Sel8 */
+
+        default:
+                DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level:"
+                              "0x%x\n", signal_levels);
+                return DDI_BUF_EMP_400MV_0DB_BDW;       /* Sel0 */
+        }
+}
+
 /* Properly updates "DP" with the correct signal levels. */
 static void
 intel_dp_set_signal_levels(struct intel_dp *intel_dp, uint32_t *DP)
@@ -2296,7 +2348,10 @@ intel_dp_set_signal_levels(struct intel_dp *intel_dp, uint32_t *DP)
         uint32_t signal_levels, mask;
         uint8_t train_set = intel_dp->train_set[0];
 
-        if (HAS_DDI(dev)) {
+        if (IS_BROADWELL(dev)) {
+                signal_levels = intel_bdw_signal_levels(train_set);
+                mask = DDI_BUF_EMP_MASK;
+        } else if (IS_HASWELL(dev)) {
                 signal_levels = intel_hsw_signal_levels(train_set);
                 mask = DDI_BUF_EMP_MASK;
         } else if (IS_VALLEYVIEW(dev)) {
@@ -3284,7 +3339,8 @@ bool intel_dpd_is_edp(struct drm_device *dev)
                 p_child = dev_priv->vbt.child_dev + i;
 
                 if (p_child->common.dvo_port == PORT_IDPD &&
-                    p_child->common.device_type == DEVICE_TYPE_eDP)
+                    (p_child->common.device_type & DEVICE_TYPE_eDP_BITS) ==
+                    (DEVICE_TYPE_eDP & DEVICE_TYPE_eDP_BITS))
                         return true;
         }
         return false;
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 9134a5464dd5..02312810374d 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -156,6 +156,17 @@ struct intel_encoder {
 struct intel_panel {
         struct drm_display_mode *fixed_mode;
         int fitting_mode;
+
+        /* backlight */
+        struct {
+                bool present;
+                u32 level;
+                u32 max;
+                bool enabled;
+                bool combination_mode;  /* gen 2/4 only */
+                bool active_low_pwm;
+                struct backlight_device *device;
+        } backlight;
 };
 
 struct intel_connector {
@@ -630,6 +641,7 @@ void intel_connector_attach_encoder(struct intel_connector *connector,
 struct drm_encoder *intel_best_encoder(struct drm_connector *connector);
 struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
                                              struct drm_crtc *crtc);
+enum pipe intel_get_pipe_from_connector(struct intel_connector *connector);
 int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
                                 struct drm_file *file_priv);
 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
@@ -803,11 +815,13 @@ void intel_pch_panel_fitting(struct intel_crtc *crtc,
 void intel_gmch_panel_fitting(struct intel_crtc *crtc,
                               struct intel_crtc_config *pipe_config,
                               int fitting_mode);
-void intel_panel_set_backlight(struct drm_device *dev, u32 level, u32 max);
+void intel_panel_set_backlight(struct intel_connector *connector, u32 level,
+                               u32 max);
 int intel_panel_setup_backlight(struct drm_connector *connector);
-void intel_panel_enable_backlight(struct drm_device *dev, enum pipe pipe);
-void intel_panel_disable_backlight(struct drm_device *dev);
-void intel_panel_destroy_backlight(struct drm_device *dev);
+void intel_panel_enable_backlight(struct intel_connector *connector);
+void intel_panel_disable_backlight(struct intel_connector *connector);
+void intel_panel_destroy_backlight(struct drm_connector *connector);
+void intel_panel_init_backlight_funcs(struct drm_device *dev);
 enum drm_connector_status intel_panel_detect(struct drm_device *dev);
 
 
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index 61cff670ff3f..d0c81b170149 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -847,7 +847,7 @@ static int hdmi_portclock_limit(struct intel_hdmi *hdmi)
 
         if (IS_G4X(dev))
                 return 165000;
-        else if (IS_HASWELL(dev))
+        else if (IS_HASWELL(dev) || INTEL_INFO(dev)->gen >= 8)
                 return 300000;
         else
                 return 225000;
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index b0ef55833087..c3b4da7895ed 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -206,7 +206,8 @@ static void intel_enable_lvds(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
         struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
-        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+        struct intel_connector *intel_connector =
+                &lvds_encoder->attached_connector->base;
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 ctl_reg, stat_reg;
 
@@ -225,13 +226,15 @@ static void intel_enable_lvds(struct intel_encoder *encoder)
         if (wait_for((I915_READ(stat_reg) & PP_ON) != 0, 1000))
                 DRM_ERROR("timed out waiting for panel to power on\n");
 
-        intel_panel_enable_backlight(dev, intel_crtc->pipe);
+        intel_panel_enable_backlight(intel_connector);
 }
 
 static void intel_disable_lvds(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
         struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
+        struct intel_connector *intel_connector =
+                &lvds_encoder->attached_connector->base;
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 ctl_reg, stat_reg;
 
@@ -243,7 +246,7 @@ static void intel_disable_lvds(struct intel_encoder *encoder)
                 stat_reg = PP_STATUS;
         }
 
-        intel_panel_disable_backlight(dev);
+        intel_panel_disable_backlight(intel_connector);
 
         I915_WRITE(ctl_reg, I915_READ(ctl_reg) & ~POWER_TARGET_ON);
         if (wait_for((I915_READ(stat_reg) & PP_ON) == 0, 1000))
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index b82050c96f3e..6506df26ac9e 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -396,6 +396,9 @@ int intel_opregion_notify_adapter(struct drm_device *dev, pci_power_t state)
 static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_connector *connector;
+        struct intel_connector *intel_connector;
+        struct intel_panel *panel;
         struct opregion_asle __iomem *asle = dev_priv->opregion.asle;
 
         DRM_DEBUG_DRIVER("bclp = 0x%08x\n", bclp);
@@ -407,10 +410,24 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
         if (bclp > 255)
                 return ASLC_BACKLIGHT_FAILED;
 
+        mutex_lock(&dev->mode_config.mutex);
+
+        /*
+         * Update backlight on all connectors that support backlight (usually
+         * only one).
+         */
         DRM_DEBUG_KMS("updating opregion backlight %d/255\n", bclp);
-        intel_panel_set_backlight(dev, bclp, 255);
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                intel_connector = to_intel_connector(connector);
+                panel = &intel_connector->panel;
+                if (panel->backlight.present)
+                        intel_panel_set_backlight(intel_connector, bclp, 255);
+        }
         iowrite32(DIV_ROUND_UP(bclp * 100, 255) | ASLE_CBLV_VALID, &asle->cblv);
 
+        mutex_unlock(&dev->mode_config.mutex);
+
+
         return 0;
 }
 
@@ -486,9 +503,13 @@ static u32 asle_isct_state(struct drm_device *dev)
         return ASLC_ISCT_STATE_FAILED;
 }
 
-void intel_opregion_asle_intr(struct drm_device *dev)
+static void asle_work(struct work_struct *work)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_opregion *opregion =
+                container_of(work, struct intel_opregion, asle_work);
+        struct drm_i915_private *dev_priv =
+                container_of(opregion, struct drm_i915_private, opregion);
+        struct drm_device *dev = dev_priv->dev;
         struct opregion_asle __iomem *asle = dev_priv->opregion.asle;
         u32 aslc_stat = 0;
         u32 aslc_req;
@@ -535,6 +556,14 @@ void intel_opregion_asle_intr(struct drm_device *dev)
         iowrite32(aslc_stat, &asle->aslc);
 }
 
+void intel_opregion_asle_intr(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (dev_priv->opregion.asle)
+                schedule_work(&dev_priv->opregion.asle_work);
+}
+
 #define ACPI_EV_DISPLAY_SWITCH (1<<0)
 #define ACPI_EV_LID            (1<<1)
 #define ACPI_EV_DOCK           (1<<2)
@@ -735,6 +764,8 @@ void intel_opregion_fini(struct drm_device *dev)
         if (opregion->asle)
                 iowrite32(ASLE_ARDY_NOT_READY, &opregion->asle->ardy);
 
+        cancel_work_sync(&dev_priv->opregion.asle_work);
+
         if (opregion->acpi) {
                 iowrite32(0, &opregion->acpi->drdy);
 
@@ -828,6 +859,10 @@ int intel_opregion_setup(struct drm_device *dev)
                 return -ENOTSUPP;
         }
 
+#ifdef CONFIG_ACPI
+        INIT_WORK(&opregion->asle_work, asle_work);
+#endif
+
         base = acpi_os_ioremap(asls, OPREGION_SIZE);
         if (!base)
                 return -ENOMEM;
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index de1518614827..e480cf41c536 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -325,214 +325,263 @@ out:
         pipe_config->gmch_pfit.lvds_border_bits = border;
 }
 
-static int is_backlight_combination_mode(struct drm_device *dev)
+static int i915_panel_invert_brightness;
+MODULE_PARM_DESC(invert_brightness, "Invert backlight brightness "
+        "(-1 force normal, 0 machine defaults, 1 force inversion), please "
+        "report PCI device ID, subsystem vendor and subsystem device ID "
+        "to dri-devel@lists.freedesktop.org, if your machine needs it. "
+        "It will then be included in an upcoming module version.");
+module_param_named(invert_brightness, i915_panel_invert_brightness, int, 0600);
+static u32 intel_panel_compute_brightness(struct intel_connector *connector,
+                                          u32 val)
 {
+        struct drm_device *dev = connector->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
 
-        if (IS_GEN4(dev))
-                return I915_READ(BLC_PWM_CTL2) & BLM_COMBINATION_MODE;
+        WARN_ON(panel->backlight.max == 0);
 
-        if (IS_GEN2(dev))
-                return I915_READ(BLC_PWM_CTL) & BLM_LEGACY_MODE;
+        if (i915_panel_invert_brightness < 0)
+                return val;
 
-        return 0;
+        if (i915_panel_invert_brightness > 0 ||
+            dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS) {
+                return panel->backlight.max - val;
+        }
+
+        return val;
 }
 
-/* XXX: query mode clock or hardware clock and program max PWM appropriately
- * when it's 0.
- */
-static u32 i915_read_blc_pwm_ctl(struct drm_device *dev)
+static u32 bdw_get_backlight(struct intel_connector *connector)
 {
+        struct drm_device *dev = connector->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 val;
-
-        WARN_ON_SMP(!spin_is_locked(&dev_priv->backlight.lock));
 
-        /* Restore the CTL value if it lost, e.g. GPU reset */
-
-        if (HAS_PCH_SPLIT(dev_priv->dev)) {
-                val = I915_READ(BLC_PWM_PCH_CTL2);
-                if (dev_priv->regfile.saveBLC_PWM_CTL2 == 0) {
-                        dev_priv->regfile.saveBLC_PWM_CTL2 = val;
-                } else if (val == 0) {
-                        val = dev_priv->regfile.saveBLC_PWM_CTL2;
-                        I915_WRITE(BLC_PWM_PCH_CTL2, val);
-                }
-        } else {
-                val = I915_READ(BLC_PWM_CTL);
-                if (dev_priv->regfile.saveBLC_PWM_CTL == 0) {
-                        dev_priv->regfile.saveBLC_PWM_CTL = val;
-                        if (INTEL_INFO(dev)->gen >= 4)
-                                dev_priv->regfile.saveBLC_PWM_CTL2 =
-                                        I915_READ(BLC_PWM_CTL2);
-                } else if (val == 0) {
-                        val = dev_priv->regfile.saveBLC_PWM_CTL;
-                        I915_WRITE(BLC_PWM_CTL, val);
-                        if (INTEL_INFO(dev)->gen >= 4)
-                                I915_WRITE(BLC_PWM_CTL2,
-                                           dev_priv->regfile.saveBLC_PWM_CTL2);
-                }
+        return I915_READ(BLC_PWM_PCH_CTL2) & BACKLIGHT_DUTY_CYCLE_MASK;
+}
 
-                if (IS_VALLEYVIEW(dev) && !val)
-                        val = 0x0f42ffff;
-        }
+static u32 pch_get_backlight(struct intel_connector *connector)
+{
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
-        return val;
+        return I915_READ(BLC_PWM_CPU_CTL) & BACKLIGHT_DUTY_CYCLE_MASK;
 }
 
-static u32 intel_panel_get_max_backlight(struct drm_device *dev)
+static u32 i9xx_get_backlight(struct intel_connector *connector)
 {
-        u32 max;
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
+        u32 val;
 
-        max = i915_read_blc_pwm_ctl(dev);
+        val = I915_READ(BLC_PWM_CTL) & BACKLIGHT_DUTY_CYCLE_MASK;
+        if (INTEL_INFO(dev)->gen < 4)
+                val >>= 1;
 
-        if (HAS_PCH_SPLIT(dev)) {
-                max >>= 16;
-        } else {
-                if (INTEL_INFO(dev)->gen < 4)
-                        max >>= 17;
-                else
-                        max >>= 16;
+        if (panel->backlight.combination_mode) {
+                u8 lbpc;
 
-                if (is_backlight_combination_mode(dev))
-                        max *= 0xff;
+                pci_read_config_byte(dev->pdev, PCI_LBPC, &lbpc);
+                val *= lbpc;
         }
 
-        DRM_DEBUG_DRIVER("max backlight PWM = %d\n", max);
-
-        return max;
+        return val;
 }
 
-static int i915_panel_invert_brightness;
-MODULE_PARM_DESC(invert_brightness, "Invert backlight brightness "
-        "(-1 force normal, 0 machine defaults, 1 force inversion), please "
-        "report PCI device ID, subsystem vendor and subsystem device ID "
-        "to dri-devel@lists.freedesktop.org, if your machine needs it. "
-        "It will then be included in an upcoming module version.");
-module_param_named(invert_brightness, i915_panel_invert_brightness, int, 0600);
-static u32 intel_panel_compute_brightness(struct drm_device *dev, u32 val)
+static u32 _vlv_get_backlight(struct drm_device *dev, enum pipe pipe)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        if (i915_panel_invert_brightness < 0)
-                return val;
+        return I915_READ(VLV_BLC_PWM_CTL(pipe)) & BACKLIGHT_DUTY_CYCLE_MASK;
+}
 
-        if (i915_panel_invert_brightness > 0 ||
-            dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS) {
-                u32 max = intel_panel_get_max_backlight(dev);
-                if (max)
-                        return max - val;
-        }
+static u32 vlv_get_backlight(struct intel_connector *connector)
+{
+        struct drm_device *dev = connector->base.dev;
+        enum pipe pipe = intel_get_pipe_from_connector(connector);
 
-        return val;
+        return _vlv_get_backlight(dev, pipe);
 }
 
-static u32 intel_panel_get_backlight(struct drm_device *dev)
+static u32 intel_panel_get_backlight(struct intel_connector *connector)
 {
+        struct drm_device *dev = connector->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 val;
         unsigned long flags;
 
-        spin_lock_irqsave(&dev_priv->backlight.lock, flags);
-
-        if (HAS_PCH_SPLIT(dev)) {
-                val = I915_READ(BLC_PWM_CPU_CTL) & BACKLIGHT_DUTY_CYCLE_MASK;
-        } else {
-                val = I915_READ(BLC_PWM_CTL) & BACKLIGHT_DUTY_CYCLE_MASK;
-                if (INTEL_INFO(dev)->gen < 4)
-                        val >>= 1;
-
-                if (is_backlight_combination_mode(dev)) {
-                        u8 lbpc;
-
-                        pci_read_config_byte(dev->pdev, PCI_LBPC, &lbpc);
-                        val *= lbpc;
-                }
-        }
+        spin_lock_irqsave(&dev_priv->backlight_lock, flags);
 
-        val = intel_panel_compute_brightness(dev, val);
+        val = dev_priv->display.get_backlight(connector);
+        val = intel_panel_compute_brightness(connector, val);
 
-        spin_unlock_irqrestore(&dev_priv->backlight.lock, flags);
+        spin_unlock_irqrestore(&dev_priv->backlight_lock, flags);
 
         DRM_DEBUG_DRIVER("get backlight PWM = %d\n", val);
         return val;
 }
 
-static void intel_pch_panel_set_backlight(struct drm_device *dev, u32 level)
+static void bdw_set_backlight(struct intel_connector *connector, u32 level)
 {
+        struct drm_device *dev = connector->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 val = I915_READ(BLC_PWM_CPU_CTL) & ~BACKLIGHT_DUTY_CYCLE_MASK;
-        I915_WRITE(BLC_PWM_CPU_CTL, val | level);
+        u32 val = I915_READ(BLC_PWM_PCH_CTL2) & ~BACKLIGHT_DUTY_CYCLE_MASK;
+        I915_WRITE(BLC_PWM_PCH_CTL2, val | level);
 }
 
-static void intel_panel_actually_set_backlight(struct drm_device *dev,
-                                               u32 level)
+static void pch_set_backlight(struct intel_connector *connector, u32 level)
 {
+        struct drm_device *dev = connector->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 tmp;
 
-        DRM_DEBUG_DRIVER("set backlight PWM = %d\n", level);
-        level = intel_panel_compute_brightness(dev, level);
+        tmp = I915_READ(BLC_PWM_CPU_CTL) & ~BACKLIGHT_DUTY_CYCLE_MASK;
+        I915_WRITE(BLC_PWM_CPU_CTL, tmp | level);
+}
+
+static void i9xx_set_backlight(struct intel_connector *connector, u32 level)
+{
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
+        u32 tmp, mask;
 
-        if (HAS_PCH_SPLIT(dev))
-                return intel_pch_panel_set_backlight(dev, level);
+        WARN_ON(panel->backlight.max == 0);
 
-        if (is_backlight_combination_mode(dev)) {
-                u32 max = intel_panel_get_max_backlight(dev);
+        if (panel->backlight.combination_mode) {
                 u8 lbpc;
 
-                /* we're screwed, but keep behaviour backwards compatible */
-                if (!max)
-                        max = 1;
-
-                lbpc = level * 0xfe / max + 1;
+                lbpc = level * 0xfe / panel->backlight.max + 1;
                 level /= lbpc;
                 pci_write_config_byte(dev->pdev, PCI_LBPC, lbpc);
         }
 
-        tmp = I915_READ(BLC_PWM_CTL);
-        if (INTEL_INFO(dev)->gen < 4)
+        if (IS_GEN4(dev)) {
+                mask = BACKLIGHT_DUTY_CYCLE_MASK;
+        } else {
                 level <<= 1;
-        tmp &= ~BACKLIGHT_DUTY_CYCLE_MASK;
+                mask = BACKLIGHT_DUTY_CYCLE_MASK_PNV;
+        }
+
+        tmp = I915_READ(BLC_PWM_CTL) & ~mask;
         I915_WRITE(BLC_PWM_CTL, tmp | level);
 }
 
+static void vlv_set_backlight(struct intel_connector *connector, u32 level)
+{
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum pipe pipe = intel_get_pipe_from_connector(connector);
+        u32 tmp;
+
+        tmp = I915_READ(VLV_BLC_PWM_CTL(pipe)) & ~BACKLIGHT_DUTY_CYCLE_MASK;
+        I915_WRITE(VLV_BLC_PWM_CTL(pipe), tmp | level);
+}
+
+static void
+intel_panel_actually_set_backlight(struct intel_connector *connector, u32 level)
+{
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        DRM_DEBUG_DRIVER("set backlight PWM = %d\n", level);
+
+        level = intel_panel_compute_brightness(connector, level);
+        dev_priv->display.set_backlight(connector, level);
+}
+
 /* set backlight brightness to level in range [0..max] */
-void intel_panel_set_backlight(struct drm_device *dev, u32 level, u32 max)
+void intel_panel_set_backlight(struct intel_connector *connector, u32 level,
+                               u32 max)
 {
+        struct drm_device *dev = connector->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
+        enum pipe pipe = intel_get_pipe_from_connector(connector);
         u32 freq;
         unsigned long flags;
 
-        spin_lock_irqsave(&dev_priv->backlight.lock, flags);
+        if (pipe == INVALID_PIPE)
+                return;
 
-        freq = intel_panel_get_max_backlight(dev);
-        if (!freq) {
-                /* we are screwed, bail out */
-                goto out;
-        }
+        spin_lock_irqsave(&dev_priv->backlight_lock, flags);
+
+        WARN_ON(panel->backlight.max == 0);
 
-        /* scale to hardware, but be careful to not overflow */
+        /* scale to hardware max, but be careful to not overflow */
+        freq = panel->backlight.max;
         if (freq < max)
                 level = level * freq / max;
         else
                 level = freq / max * level;
 
-        dev_priv->backlight.level = level;
-        if (dev_priv->backlight.device)
-                dev_priv->backlight.device->props.brightness = level;
+        panel->backlight.level = level;
+        if (panel->backlight.device)
+                panel->backlight.device->props.brightness = level;
 
-        if (dev_priv->backlight.enabled)
-                intel_panel_actually_set_backlight(dev, level);
-out:
-        spin_unlock_irqrestore(&dev_priv->backlight.lock, flags);
+        if (panel->backlight.enabled)
+                intel_panel_actually_set_backlight(connector, level);
+
+        spin_unlock_irqrestore(&dev_priv->backlight_lock, flags);
+}
+
+static void pch_disable_backlight(struct intel_connector *connector)
+{
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 tmp;
+
+        intel_panel_actually_set_backlight(connector, 0);
+
+        tmp = I915_READ(BLC_PWM_CPU_CTL2);
+        I915_WRITE(BLC_PWM_CPU_CTL2, tmp & ~BLM_PWM_ENABLE);
+
+        tmp = I915_READ(BLC_PWM_PCH_CTL1);
+        I915_WRITE(BLC_PWM_PCH_CTL1, tmp & ~BLM_PCH_PWM_ENABLE);
+}
+
+static void i9xx_disable_backlight(struct intel_connector *connector)
+{
+        intel_panel_actually_set_backlight(connector, 0);
 }
 
-void intel_panel_disable_backlight(struct drm_device *dev)
+static void i965_disable_backlight(struct intel_connector *connector)
 {
+        struct drm_device *dev = connector->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 tmp;
+
+        intel_panel_actually_set_backlight(connector, 0);
+
+        tmp = I915_READ(BLC_PWM_CTL2);
+        I915_WRITE(BLC_PWM_CTL2, tmp & ~BLM_PWM_ENABLE);
+}
+
+static void vlv_disable_backlight(struct intel_connector *connector)
+{
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum pipe pipe = intel_get_pipe_from_connector(connector);
+        u32 tmp;
+
+        intel_panel_actually_set_backlight(connector, 0);
+
+        tmp = I915_READ(VLV_BLC_PWM_CTL2(pipe));
+        I915_WRITE(VLV_BLC_PWM_CTL2(pipe), tmp & ~BLM_PWM_ENABLE);
+}
+
+void intel_panel_disable_backlight(struct intel_connector *connector)
+{
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
+        enum pipe pipe = intel_get_pipe_from_connector(connector);
         unsigned long flags;
 
+        if (pipe == INVALID_PIPE)
+                return;
+
         /*
          * Do not disable backlight on the vgaswitcheroo path. When switching
          * away from i915, the other client may depend on i915 to handle the
@@ -544,116 +593,215 @@ void intel_panel_disable_backlight(struct drm_device *dev)
                 return;
         }
 
-        spin_lock_irqsave(&dev_priv->backlight.lock, flags);
+        spin_lock_irqsave(&dev_priv->backlight_lock, flags);
 
-        dev_priv->backlight.enabled = false;
-        intel_panel_actually_set_backlight(dev, 0);
+        panel->backlight.enabled = false;
+        dev_priv->display.disable_backlight(connector);
 
-        if (INTEL_INFO(dev)->gen >= 4) {
-                uint32_t reg, tmp;
+        spin_unlock_irqrestore(&dev_priv->backlight_lock, flags);
+}
+
+static void bdw_enable_backlight(struct intel_connector *connector)
+{
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
+        u32 pch_ctl1, pch_ctl2;
+
+        pch_ctl1 = I915_READ(BLC_PWM_PCH_CTL1);
+        if (pch_ctl1 & BLM_PCH_PWM_ENABLE) {
+                DRM_DEBUG_KMS("pch backlight already enabled\n");
+                pch_ctl1 &= ~BLM_PCH_PWM_ENABLE;
+                I915_WRITE(BLC_PWM_PCH_CTL1, pch_ctl1);
+        }
 
-                reg = HAS_PCH_SPLIT(dev) ? BLC_PWM_CPU_CTL2 : BLC_PWM_CTL2;
+        pch_ctl2 = panel->backlight.max << 16;
+        I915_WRITE(BLC_PWM_PCH_CTL2, pch_ctl2);
 
-                I915_WRITE(reg, I915_READ(reg) & ~BLM_PWM_ENABLE);
+        pch_ctl1 = 0;
+        if (panel->backlight.active_low_pwm)
+                pch_ctl1 |= BLM_PCH_POLARITY;
 
-                if (HAS_PCH_SPLIT(dev)) {
-                        tmp = I915_READ(BLC_PWM_PCH_CTL1);
-                        tmp &= ~BLM_PCH_PWM_ENABLE;
-                        I915_WRITE(BLC_PWM_PCH_CTL1, tmp);
-                }
-        }
+        /* BDW always uses the pch pwm controls. */
+        pch_ctl1 |= BLM_PCH_OVERRIDE_ENABLE;
+
+        I915_WRITE(BLC_PWM_PCH_CTL1, pch_ctl1);
+        POSTING_READ(BLC_PWM_PCH_CTL1);
+        I915_WRITE(BLC_PWM_PCH_CTL1, pch_ctl1 | BLM_PCH_PWM_ENABLE);
 
-        spin_unlock_irqrestore(&dev_priv->backlight.lock, flags);
+        /* This won't stick until the above enable. */
+        intel_panel_actually_set_backlight(connector, panel->backlight.level);
 }
 
-void intel_panel_enable_backlight(struct drm_device *dev,
-                                  enum pipe pipe)
+static void pch_enable_backlight(struct intel_connector *connector)
 {
+        struct drm_device *dev = connector->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
+        enum pipe pipe = intel_get_pipe_from_connector(connector);
         enum transcoder cpu_transcoder =
                 intel_pipe_to_cpu_transcoder(dev_priv, pipe);
-        unsigned long flags;
-
-        DRM_DEBUG_KMS("pipe %c\n", pipe_name(pipe));
+        u32 cpu_ctl2, pch_ctl1, pch_ctl2;
 
-        spin_lock_irqsave(&dev_priv->backlight.lock, flags);
+        cpu_ctl2 = I915_READ(BLC_PWM_CPU_CTL2);
+        if (cpu_ctl2 & BLM_PWM_ENABLE) {
+                WARN(1, "cpu backlight already enabled\n");
+                cpu_ctl2 &= ~BLM_PWM_ENABLE;
+                I915_WRITE(BLC_PWM_CPU_CTL2, cpu_ctl2);
+        }
 
-        if (dev_priv->backlight.level == 0) {
-                dev_priv->backlight.level = intel_panel_get_max_backlight(dev);
-                if (dev_priv->backlight.device)
-                        dev_priv->backlight.device->props.brightness =
-                                dev_priv->backlight.level;
+        pch_ctl1 = I915_READ(BLC_PWM_PCH_CTL1);
+        if (pch_ctl1 & BLM_PCH_PWM_ENABLE) {
+                DRM_DEBUG_KMS("pch backlight already enabled\n");
+                pch_ctl1 &= ~BLM_PCH_PWM_ENABLE;
+                I915_WRITE(BLC_PWM_PCH_CTL1, pch_ctl1);
         }
 
-        if (INTEL_INFO(dev)->gen >= 4) {
-                uint32_t reg, tmp;
+        if (cpu_transcoder == TRANSCODER_EDP)
+                cpu_ctl2 = BLM_TRANSCODER_EDP;
+        else
+                cpu_ctl2 = BLM_PIPE(cpu_transcoder);
+        I915_WRITE(BLC_PWM_CPU_CTL2, cpu_ctl2);
+        POSTING_READ(BLC_PWM_CPU_CTL2);
+        I915_WRITE(BLC_PWM_CPU_CTL2, cpu_ctl2 | BLM_PWM_ENABLE);
 
-                reg = HAS_PCH_SPLIT(dev) ? BLC_PWM_CPU_CTL2 : BLC_PWM_CTL2;
+        /* This won't stick until the above enable. */
+        intel_panel_actually_set_backlight(connector, panel->backlight.level);
 
+        pch_ctl2 = panel->backlight.max << 16;
+        I915_WRITE(BLC_PWM_PCH_CTL2, pch_ctl2);
 
-                tmp = I915_READ(reg);
+        pch_ctl1 = 0;
+        if (panel->backlight.active_low_pwm)
+                pch_ctl1 |= BLM_PCH_POLARITY;
 
-                /* Note that this can also get called through dpms changes. And
-                 * we don't track the backlight dpms state, hence check whether
-                 * we have to do anything first. */
-                if (tmp & BLM_PWM_ENABLE)
-                        goto set_level;
+        I915_WRITE(BLC_PWM_PCH_CTL1, pch_ctl1);
+        POSTING_READ(BLC_PWM_PCH_CTL1);
+        I915_WRITE(BLC_PWM_PCH_CTL1, pch_ctl1 | BLM_PCH_PWM_ENABLE);
+}
 
-                if (INTEL_INFO(dev)->num_pipes == 3)
-                        tmp &= ~BLM_PIPE_SELECT_IVB;
-                else
-                        tmp &= ~BLM_PIPE_SELECT;
+static void i9xx_enable_backlight(struct intel_connector *connector)
+{
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
+        u32 ctl, freq;
 
-                if (cpu_transcoder == TRANSCODER_EDP)
-                        tmp |= BLM_TRANSCODER_EDP;
-                else
-                        tmp |= BLM_PIPE(cpu_transcoder);
-                tmp &= ~BLM_PWM_ENABLE;
-
-                I915_WRITE(reg, tmp);
-                POSTING_READ(reg);
-                I915_WRITE(reg, tmp | BLM_PWM_ENABLE);
-
-                if (HAS_PCH_SPLIT(dev) &&
-                    !(dev_priv->quirks & QUIRK_NO_PCH_PWM_ENABLE)) {
-                        tmp = I915_READ(BLC_PWM_PCH_CTL1);
-                        tmp |= BLM_PCH_PWM_ENABLE;
-                        tmp &= ~BLM_PCH_OVERRIDE_ENABLE;
-                        I915_WRITE(BLC_PWM_PCH_CTL1, tmp);
-                }
+        ctl = I915_READ(BLC_PWM_CTL);
+        if (ctl & BACKLIGHT_DUTY_CYCLE_MASK_PNV) {
+                WARN(1, "backlight already enabled\n");
+                I915_WRITE(BLC_PWM_CTL, 0);
         }
 
-set_level:
-        /* Call below after setting BLC_PWM_CPU_CTL2 and BLC_PWM_PCH_CTL1.
-         * BLC_PWM_CPU_CTL may be cleared to zero automatically when these
-         * registers are set.
-         */
-        dev_priv->backlight.enabled = true;
-        intel_panel_actually_set_backlight(dev, dev_priv->backlight.level);
+        freq = panel->backlight.max;
+        if (panel->backlight.combination_mode)
+                freq /= 0xff;
+
+        ctl = freq << 17;
+        if (IS_GEN2(dev) && panel->backlight.combination_mode)
+                ctl |= BLM_LEGACY_MODE;
+        if (IS_PINEVIEW(dev) && panel->backlight.active_low_pwm)
+                ctl |= BLM_POLARITY_PNV;
 
-        spin_unlock_irqrestore(&dev_priv->backlight.lock, flags);
+        I915_WRITE(BLC_PWM_CTL, ctl);
+        POSTING_READ(BLC_PWM_CTL);
+
+        /* XXX: combine this into above write? */
+        intel_panel_actually_set_backlight(connector, panel->backlight.level);
 }
 
-/* FIXME: use VBT vals to init PWM_CTL and PWM_CTL2 correctly */
-static void intel_panel_init_backlight_regs(struct drm_device *dev)
+static void i965_enable_backlight(struct intel_connector *connector)
 {
+        struct drm_device *dev = connector->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
+        enum pipe pipe = intel_get_pipe_from_connector(connector);
+        u32 ctl, ctl2, freq;
+
+        ctl2 = I915_READ(BLC_PWM_CTL2);
+        if (ctl2 & BLM_PWM_ENABLE) {
+                WARN(1, "backlight already enabled\n");
+                ctl2 &= ~BLM_PWM_ENABLE;
+                I915_WRITE(BLC_PWM_CTL2, ctl2);
+        }
+
+        freq = panel->backlight.max;
+        if (panel->backlight.combination_mode)
+                freq /= 0xff;
+
+        ctl = freq << 16;
+        I915_WRITE(BLC_PWM_CTL, ctl);
 
-        if (IS_VALLEYVIEW(dev)) {
-                u32 cur_val = I915_READ(BLC_PWM_CTL) &
-                        BACKLIGHT_DUTY_CYCLE_MASK;
-                I915_WRITE(BLC_PWM_CTL, (0xf42 << 16) | cur_val);
+        /* XXX: combine this into above write? */
+        intel_panel_actually_set_backlight(connector, panel->backlight.level);
+
+        ctl2 = BLM_PIPE(pipe);
+        if (panel->backlight.combination_mode)
+                ctl2 |= BLM_COMBINATION_MODE;
+        if (panel->backlight.active_low_pwm)
+                ctl2 |= BLM_POLARITY_I965;
+        I915_WRITE(BLC_PWM_CTL2, ctl2);
+        POSTING_READ(BLC_PWM_CTL2);
+        I915_WRITE(BLC_PWM_CTL2, ctl2 | BLM_PWM_ENABLE);
+}
+
+static void vlv_enable_backlight(struct intel_connector *connector)
+{
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
+        enum pipe pipe = intel_get_pipe_from_connector(connector);
+        u32 ctl, ctl2;
+
+        ctl2 = I915_READ(VLV_BLC_PWM_CTL2(pipe));
+        if (ctl2 & BLM_PWM_ENABLE) {
+                WARN(1, "backlight already enabled\n");
+                ctl2 &= ~BLM_PWM_ENABLE;
+                I915_WRITE(VLV_BLC_PWM_CTL2(pipe), ctl2);
         }
+
+        ctl = panel->backlight.max << 16;
+        I915_WRITE(VLV_BLC_PWM_CTL(pipe), ctl);
+
+        /* XXX: combine this into above write? */
+        intel_panel_actually_set_backlight(connector, panel->backlight.level);
+
+        ctl2 = 0;
+        if (panel->backlight.active_low_pwm)
+                ctl2 |= BLM_POLARITY_I965;
+        I915_WRITE(VLV_BLC_PWM_CTL2(pipe), ctl2);
+        POSTING_READ(VLV_BLC_PWM_CTL2(pipe));
+        I915_WRITE(VLV_BLC_PWM_CTL2(pipe), ctl2 | BLM_PWM_ENABLE);
 }
 
-static void intel_panel_init_backlight(struct drm_device *dev)
+void intel_panel_enable_backlight(struct intel_connector *connector)
 {
+        struct drm_device *dev = connector->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
+        enum pipe pipe = intel_get_pipe_from_connector(connector);
+        unsigned long flags;
+
+        if (pipe == INVALID_PIPE)
+                return;
+
+        DRM_DEBUG_KMS("pipe %c\n", pipe_name(pipe));
+
+        spin_lock_irqsave(&dev_priv->backlight_lock, flags);
+
+        WARN_ON(panel->backlight.max == 0);
+
+        if (panel->backlight.level == 0) {
+                panel->backlight.level = panel->backlight.max;
+                if (panel->backlight.device)
+                        panel->backlight.device->props.brightness =
+                                panel->backlight.level;
+        }
 
-        intel_panel_init_backlight_regs(dev);
+        dev_priv->display.enable_backlight(connector);
+        panel->backlight.enabled = true;
 
-        dev_priv->backlight.level = intel_panel_get_backlight(dev);
-        dev_priv->backlight.enabled = dev_priv->backlight.level != 0;
+        spin_unlock_irqrestore(&dev_priv->backlight_lock, flags);
 }
 
 enum drm_connector_status
@@ -679,85 +827,320 @@ intel_panel_detect(struct drm_device *dev)
 }
 
 #if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE)
-static int intel_panel_update_status(struct backlight_device *bd)
+static int intel_backlight_device_update_status(struct backlight_device *bd)
 {
-        struct drm_device *dev = bl_get_data(bd);
+        struct intel_connector *connector = bl_get_data(bd);
+        struct drm_device *dev = connector->base.dev;
+
+        mutex_lock(&dev->mode_config.mutex);
         DRM_DEBUG_KMS("updating intel_backlight, brightness=%d/%d\n",
                       bd->props.brightness, bd->props.max_brightness);
-        intel_panel_set_backlight(dev, bd->props.brightness,
+        intel_panel_set_backlight(connector, bd->props.brightness,
                                   bd->props.max_brightness);
+        mutex_unlock(&dev->mode_config.mutex);
         return 0;
 }
 
-static int intel_panel_get_brightness(struct backlight_device *bd)
+static int intel_backlight_device_get_brightness(struct backlight_device *bd)
 {
-        struct drm_device *dev = bl_get_data(bd);
-        return intel_panel_get_backlight(dev);
+        struct intel_connector *connector = bl_get_data(bd);
+        struct drm_device *dev = connector->base.dev;
+        int ret;
+
+        mutex_lock(&dev->mode_config.mutex);
+        ret = intel_panel_get_backlight(connector);
+        mutex_unlock(&dev->mode_config.mutex);
+
+        return ret;
 }
 
-static const struct backlight_ops intel_panel_bl_ops = {
-        .update_status = intel_panel_update_status,
-        .get_brightness = intel_panel_get_brightness,
+static const struct backlight_ops intel_backlight_device_ops = {
+        .update_status = intel_backlight_device_update_status,
+        .get_brightness = intel_backlight_device_get_brightness,
 };
 
-int intel_panel_setup_backlight(struct drm_connector *connector)
+static int intel_backlight_device_register(struct intel_connector *connector)
 {
-        struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
         struct backlight_properties props;
-        unsigned long flags;
 
-        intel_panel_init_backlight(dev);
-
-        if (WARN_ON(dev_priv->backlight.device))
+        if (WARN_ON(panel->backlight.device))
                 return -ENODEV;
 
+        BUG_ON(panel->backlight.max == 0);
+
         memset(&props, 0, sizeof(props));
         props.type = BACKLIGHT_RAW;
-        props.brightness = dev_priv->backlight.level;
-
-        spin_lock_irqsave(&dev_priv->backlight.lock, flags);
-        props.max_brightness = intel_panel_get_max_backlight(dev);
-        spin_unlock_irqrestore(&dev_priv->backlight.lock, flags);
+        props.brightness = panel->backlight.level;
+        props.max_brightness = panel->backlight.max;
 
-        if (props.max_brightness == 0) {
-                DRM_DEBUG_DRIVER("Failed to get maximum backlight value\n");
-                return -ENODEV;
-        }
-        dev_priv->backlight.device =
+        /*
+         * Note: using the same name independent of the connector prevents
+         * registration of multiple backlight devices in the driver.
+         */
+        panel->backlight.device =
                 backlight_device_register("intel_backlight",
-                                          &connector->kdev, dev,
-                                          &intel_panel_bl_ops, &props);
+                                          connector->base.kdev,
+                                          connector,
+                                          &intel_backlight_device_ops, &props);
 
-        if (IS_ERR(dev_priv->backlight.device)) {
+        if (IS_ERR(panel->backlight.device)) {
                 DRM_ERROR("Failed to register backlight: %ld\n",
-                          PTR_ERR(dev_priv->backlight.device));
-                dev_priv->backlight.device = NULL;
+                          PTR_ERR(panel->backlight.device));
+                panel->backlight.device = NULL;
                 return -ENODEV;
         }
         return 0;
 }
 
-void intel_panel_destroy_backlight(struct drm_device *dev)
+static void intel_backlight_device_unregister(struct intel_connector *connector)
 {
+        struct intel_panel *panel = &connector->panel;
+
+        if (panel->backlight.device) {
+                backlight_device_unregister(panel->backlight.device);
+                panel->backlight.device = NULL;
+        }
+}
+#else /* CONFIG_BACKLIGHT_CLASS_DEVICE */
+static int intel_backlight_device_register(struct intel_connector *connector)
+{
+        return 0;
+}
+static void intel_backlight_device_unregister(struct intel_connector *connector)
+{
+}
+#endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */
+
+/*
+ * Note: The setup hooks can't assume pipe is set!
+ *
+ * XXX: Query mode clock or hardware clock and program PWM modulation frequency
+ * appropriately when it's 0. Use VBT and/or sane defaults.
+ */
+static int bdw_setup_backlight(struct intel_connector *connector)
+{
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
+        u32 pch_ctl1, pch_ctl2, val;
+
+        pch_ctl1 = I915_READ(BLC_PWM_PCH_CTL1);
+        panel->backlight.active_low_pwm = pch_ctl1 & BLM_PCH_POLARITY;
+
+        pch_ctl2 = I915_READ(BLC_PWM_PCH_CTL2);
+        panel->backlight.max = pch_ctl2 >> 16;
+        if (!panel->backlight.max)
+                return -ENODEV;
+
+        val = bdw_get_backlight(connector);
+        panel->backlight.level = intel_panel_compute_brightness(connector, val);
+
+        panel->backlight.enabled = (pch_ctl1 & BLM_PCH_PWM_ENABLE) &&
+                panel->backlight.level != 0;
+
+        return 0;
+}
+
+static int pch_setup_backlight(struct intel_connector *connector)
+{
+        struct drm_device *dev = connector->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        if (dev_priv->backlight.device) {
-                backlight_device_unregister(dev_priv->backlight.device);
-                dev_priv->backlight.device = NULL;
+        struct intel_panel *panel = &connector->panel;
+        u32 cpu_ctl2, pch_ctl1, pch_ctl2, val;
+
+        pch_ctl1 = I915_READ(BLC_PWM_PCH_CTL1);
+        panel->backlight.active_low_pwm = pch_ctl1 & BLM_PCH_POLARITY;
+
+        pch_ctl2 = I915_READ(BLC_PWM_PCH_CTL2);
+        panel->backlight.max = pch_ctl2 >> 16;
+        if (!panel->backlight.max)
+                return -ENODEV;
+
+        val = pch_get_backlight(connector);
+        panel->backlight.level = intel_panel_compute_brightness(connector, val);
+
+        cpu_ctl2 = I915_READ(BLC_PWM_CPU_CTL2);
+        panel->backlight.enabled = (cpu_ctl2 & BLM_PWM_ENABLE) &&
+                (pch_ctl1 & BLM_PCH_PWM_ENABLE) && panel->backlight.level != 0;
+
+        return 0;
+}
+
+static int i9xx_setup_backlight(struct intel_connector *connector)
+{
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
+        u32 ctl, val;
+
+        ctl = I915_READ(BLC_PWM_CTL);
+
+        if (IS_GEN2(dev))
+                panel->backlight.combination_mode = ctl & BLM_LEGACY_MODE;
+
+        if (IS_PINEVIEW(dev))
+                panel->backlight.active_low_pwm = ctl & BLM_POLARITY_PNV;
+
+        panel->backlight.max = ctl >> 17;
+        if (panel->backlight.combination_mode)
+                panel->backlight.max *= 0xff;
+
+        if (!panel->backlight.max)
+                return -ENODEV;
+
+        val = i9xx_get_backlight(connector);
+        panel->backlight.level = intel_panel_compute_brightness(connector, val);
+
+        panel->backlight.enabled = panel->backlight.level != 0;
+
+        return 0;
+}
+
+static int i965_setup_backlight(struct intel_connector *connector)
+{
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
+        u32 ctl, ctl2, val;
+
+        ctl2 = I915_READ(BLC_PWM_CTL2);
+        panel->backlight.combination_mode = ctl2 & BLM_COMBINATION_MODE;
+        panel->backlight.active_low_pwm = ctl2 & BLM_POLARITY_I965;
+
+        ctl = I915_READ(BLC_PWM_CTL);
+        panel->backlight.max = ctl >> 16;
+        if (panel->backlight.combination_mode)
+                panel->backlight.max *= 0xff;
+
+        if (!panel->backlight.max)
+                return -ENODEV;
+
+        val = i9xx_get_backlight(connector);
+        panel->backlight.level = intel_panel_compute_brightness(connector, val);
+
+        panel->backlight.enabled = (ctl2 & BLM_PWM_ENABLE) &&
+                panel->backlight.level != 0;
+
+        return 0;
+}
+
+static int vlv_setup_backlight(struct intel_connector *connector)
+{
+        struct drm_device *dev = connector->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_panel *panel = &connector->panel;
+        enum pipe pipe;
+        u32 ctl, ctl2, val;
+
+        for_each_pipe(pipe) {
+                u32 cur_val = I915_READ(VLV_BLC_PWM_CTL(pipe));
+
+                /* Skip if the modulation freq is already set */
+                if (cur_val & ~BACKLIGHT_DUTY_CYCLE_MASK)
+                        continue;
+
+                cur_val &= BACKLIGHT_DUTY_CYCLE_MASK;
+                I915_WRITE(VLV_BLC_PWM_CTL(pipe), (0xf42 << 16) |
+                           cur_val);
         }
+
+        ctl2 = I915_READ(VLV_BLC_PWM_CTL2(PIPE_A));
+        panel->backlight.active_low_pwm = ctl2 & BLM_POLARITY_I965;
+
+        ctl = I915_READ(VLV_BLC_PWM_CTL(PIPE_A));
+        panel->backlight.max = ctl >> 16;
+        if (!panel->backlight.max)
+                return -ENODEV;
+
+        val = _vlv_get_backlight(dev, PIPE_A);
+        panel->backlight.level = intel_panel_compute_brightness(connector, val);
+
+        panel->backlight.enabled = (ctl2 & BLM_PWM_ENABLE) &&
+                panel->backlight.level != 0;
+
+        return 0;
 }
-#else
+
 int intel_panel_setup_backlight(struct drm_connector *connector)
 {
-        intel_panel_init_backlight(connector->dev);
+        struct drm_device *dev = connector->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_connector *intel_connector = to_intel_connector(connector);
+        struct intel_panel *panel = &intel_connector->panel;
+        unsigned long flags;
+        int ret;
+
+        /* set level and max in panel struct */
+        spin_lock_irqsave(&dev_priv->backlight_lock, flags);
+        ret = dev_priv->display.setup_backlight(intel_connector);
+        spin_unlock_irqrestore(&dev_priv->backlight_lock, flags);
+
+        if (ret) {
+                DRM_DEBUG_KMS("failed to setup backlight for connector %s\n",
+                              drm_get_connector_name(connector));
+                return ret;
+        }
+
+        intel_backlight_device_register(intel_connector);
+
+        panel->backlight.present = true;
+
+        DRM_DEBUG_KMS("backlight initialized, %s, brightness %u/%u, "
+                      "sysfs interface %sregistered\n",
+                      panel->backlight.enabled ? "enabled" : "disabled",
+                      panel->backlight.level, panel->backlight.max,
+                      panel->backlight.device ? "" : "not ");
+
         return 0;
 }
 
-void intel_panel_destroy_backlight(struct drm_device *dev)
+void intel_panel_destroy_backlight(struct drm_connector *connector)
 {
-        return;
+        struct intel_connector *intel_connector = to_intel_connector(connector);
+        struct intel_panel *panel = &intel_connector->panel;
+
+        panel->backlight.present = false;
+        intel_backlight_device_unregister(intel_connector);
+}
+
+/* Set up chip specific backlight functions */
+void intel_panel_init_backlight_funcs(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (IS_BROADWELL(dev)) {
+                dev_priv->display.setup_backlight = bdw_setup_backlight;
+                dev_priv->display.enable_backlight = bdw_enable_backlight;
+                dev_priv->display.disable_backlight = pch_disable_backlight;
+                dev_priv->display.set_backlight = bdw_set_backlight;
+                dev_priv->display.get_backlight = bdw_get_backlight;
+        } else if (HAS_PCH_SPLIT(dev)) {
+                dev_priv->display.setup_backlight = pch_setup_backlight;
+                dev_priv->display.enable_backlight = pch_enable_backlight;
+                dev_priv->display.disable_backlight = pch_disable_backlight;
+                dev_priv->display.set_backlight = pch_set_backlight;
+                dev_priv->display.get_backlight = pch_get_backlight;
+        } else if (IS_VALLEYVIEW(dev)) {
+                dev_priv->display.setup_backlight = vlv_setup_backlight;
+                dev_priv->display.enable_backlight = vlv_enable_backlight;
+                dev_priv->display.disable_backlight = vlv_disable_backlight;
+                dev_priv->display.set_backlight = vlv_set_backlight;
+                dev_priv->display.get_backlight = vlv_get_backlight;
+        } else if (IS_GEN4(dev)) {
+                dev_priv->display.setup_backlight = i965_setup_backlight;
+                dev_priv->display.enable_backlight = i965_enable_backlight;
+                dev_priv->display.disable_backlight = i965_disable_backlight;
+                dev_priv->display.set_backlight = i9xx_set_backlight;
+                dev_priv->display.get_backlight = i9xx_get_backlight;
+        } else {
+                dev_priv->display.setup_backlight = i9xx_setup_backlight;
+                dev_priv->display.enable_backlight = i9xx_enable_backlight;
+                dev_priv->display.disable_backlight = i9xx_disable_backlight;
+                dev_priv->display.set_backlight = i9xx_set_backlight;
+                dev_priv->display.get_backlight = i9xx_get_backlight;
+        }
 }
-#endif
 
 int intel_panel_init(struct intel_panel *panel,
                      struct drm_display_mode *fixed_mode)
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 542b4448ccb7..172efa0bfb86 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -2291,7 +2291,9 @@ static uint32_t ilk_compute_fbc_wm(const struct hsw_pipe_wm_parameters *params,
 
 static unsigned int ilk_display_fifo_size(const struct drm_device *dev)
 {
-        if (INTEL_INFO(dev)->gen >= 7)
+        if (INTEL_INFO(dev)->gen >= 8)
+                return 3072;
+        else if (INTEL_INFO(dev)->gen >= 7)
                 return 768;
         else
                 return 512;
@@ -2336,7 +2338,9 @@ static unsigned int ilk_plane_wm_max(const struct drm_device *dev,
         }
 
         /* clamp to max that the registers can hold */
-        if (INTEL_INFO(dev)->gen >= 7)
+        if (INTEL_INFO(dev)->gen >= 8)
+                max = level == 0 ? 255 : 2047;
+        else if (INTEL_INFO(dev)->gen >= 7)
                 /* IVB/HSW primary/sprite plane watermarks */
                 max = level == 0 ? 127 : 1023;
         else if (!is_sprite)
@@ -2366,10 +2370,13 @@ static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
 }
 
 /* Calculate the maximum FBC watermark */
-static unsigned int ilk_fbc_wm_max(void)
+static unsigned int ilk_fbc_wm_max(struct drm_device *dev)
 {
         /* max that registers can hold */
-        return 15;
+        if (INTEL_INFO(dev)->gen >= 8)
+                return 31;
+        else
+                return 15;
 }
 
 static void ilk_compute_wm_maximums(struct drm_device *dev,
@@ -2381,7 +2388,7 @@ static void ilk_compute_wm_maximums(struct drm_device *dev,
         max->pri = ilk_plane_wm_max(dev, level, config, ddb_partitioning, false);
         max->spr = ilk_plane_wm_max(dev, level, config, ddb_partitioning, true);
         max->cur = ilk_cursor_wm_max(dev, level, config);
-        max->fbc = ilk_fbc_wm_max();
+        max->fbc = ilk_fbc_wm_max(dev);
 }
 
 static bool ilk_validate_wm_level(int level,
@@ -2722,10 +2729,18 @@ static void hsw_compute_wm_results(struct drm_device *dev,
                 if (!r->enable)
                         break;
 
-                results->wm_lp[wm_lp - 1] = HSW_WM_LP_VAL(level * 2,
-                                                          r->fbc_val,
-                                                          r->pri_val,
-                                                          r->cur_val);
+                results->wm_lp[wm_lp - 1] = WM3_LP_EN |
+                        ((level * 2) << WM1_LP_LATENCY_SHIFT) |
+                        (r->pri_val << WM1_LP_SR_SHIFT) |
+                        r->cur_val;
+
+                if (INTEL_INFO(dev)->gen >= 8)
+                        results->wm_lp[wm_lp - 1] |=
+                                r->fbc_val << WM1_LP_FBC_SHIFT_BDW;
+                else
+                        results->wm_lp[wm_lp - 1] |=
+                                r->fbc_val << WM1_LP_FBC_SHIFT;
+
                 results->wm_lp_spr[wm_lp - 1] = r->spr_val;
         }
 
@@ -3710,6 +3725,78 @@ static void gen6_enable_rps_interrupts(struct drm_device *dev)
         I915_WRITE(GEN6_PMINTRMSK, ~enabled_intrs);
 }
 
+static void gen8_enable_rps(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring;
+        uint32_t rc6_mask = 0, rp_state_cap;
+        int unused;
+
+        /* 1a: Software RC state - RC0 */
+        I915_WRITE(GEN6_RC_STATE, 0);
+
+        /* 1c & 1d: Get forcewake during program sequence. Although the driver
+         * hasn't enabled a state yet where we need forcewake, BIOS may have.*/
+        gen6_gt_force_wake_get(dev_priv);
+
+        /* 2a: Disable RC states. */
+        I915_WRITE(GEN6_RC_CONTROL, 0);
+
+        rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
+
+        /* 2b: Program RC6 thresholds.*/
+        I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
+        I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
+        I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
+        for_each_ring(ring, dev_priv, unused)
+                I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
+        I915_WRITE(GEN6_RC_SLEEP, 0);
+        I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
+
+        /* 3: Enable RC6 */
+        if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
+                rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
+        DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off");
+        I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
+                        GEN6_RC_CTL_EI_MODE(1) |
+                        rc6_mask);
+
+        /* 4 Program defaults and thresholds for RPS*/
+        I915_WRITE(GEN6_RPNSWREQ, HSW_FREQUENCY(10)); /* Request 500 MHz */
+        I915_WRITE(GEN6_RC_VIDEO_FREQ, HSW_FREQUENCY(12)); /* Request 600 MHz */
+        /* NB: Docs say 1s, and 1000000 - which aren't equivalent */
+        I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 100000000 / 128); /* 1 second timeout */
+
+        /* Docs recommend 900MHz, and 300 MHz respectively */
+        I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
+                   dev_priv->rps.max_delay << 24 |
+                   dev_priv->rps.min_delay << 16);
+
+        I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
+        I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
+        I915_WRITE(GEN6_RP_UP_EI, 66000); /* 84.48ms, XXX: random? */
+        I915_WRITE(GEN6_RP_DOWN_EI, 350000); /* 448ms, XXX: random? */
+
+        I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
+
+        /* 5: Enable RPS */
+        I915_WRITE(GEN6_RP_CONTROL,
+                   GEN6_RP_MEDIA_TURBO |
+                   GEN6_RP_MEDIA_HW_NORMAL_MODE |
+                   GEN6_RP_MEDIA_IS_GFX |
+                   GEN6_RP_ENABLE |
+                   GEN6_RP_UP_BUSY_AVG |
+                   GEN6_RP_DOWN_IDLE_AVG);
+
+        /* 6: Ring frequency + overclocking (our driver does this later */
+
+        gen6_set_rps(dev, (I915_READ(GEN6_GT_PERF_STATUS) & 0xff00) >> 8);
+
+        gen6_enable_rps_interrupts(dev);
+
+        gen6_gt_force_wake_put(dev_priv);
+}
+
 static void gen6_enable_rps(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -3872,7 +3959,10 @@ void gen6_update_ring_freq(struct drm_device *dev)
                 int diff = dev_priv->rps.max_delay - gpu_freq;
                 unsigned int ia_freq = 0, ring_freq = 0;
 
-                if (IS_HASWELL(dev)) {
+                if (INTEL_INFO(dev)->gen >= 8) {
+                        /* max(2 * GT, DDR). NB: GT is 50MHz units */
+                        ring_freq = max(min_ring_freq, gpu_freq);
+                } else if (IS_HASWELL(dev)) {
                         ring_freq = mult_frac(gpu_freq, 5, 4);
                         ring_freq = max(min_ring_freq, ring_freq);
                         /* leave ia_freq as the default, chosen by cpufreq */
@@ -4818,6 +4908,9 @@ static void intel_gen6_powersave_work(struct work_struct *work)
 
         if (IS_VALLEYVIEW(dev)) {
                 valleyview_enable_rps(dev);
+        } else if (IS_BROADWELL(dev)) {
+                gen8_enable_rps(dev);
+                gen6_update_ring_freq(dev);
         } else {
                 gen6_enable_rps(dev);
                 gen6_update_ring_freq(dev);
@@ -5126,6 +5219,50 @@ static void lpt_suspend_hw(struct drm_device *dev)
         }
 }
 
+static void gen8_init_clock_gating(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum pipe i;
+
+        I915_WRITE(WM3_LP_ILK, 0);
+        I915_WRITE(WM2_LP_ILK, 0);
+        I915_WRITE(WM1_LP_ILK, 0);
+
+        /* FIXME(BDW): Check all the w/a, some might only apply to
+         * pre-production hw. */
+
+        WARN(!i915_preliminary_hw_support,
+             "GEN8_CENTROID_PIXEL_OPT_DIS not be needed for production\n");
+        I915_WRITE(HALF_SLICE_CHICKEN3,
+                   _MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS));
+        I915_WRITE(HALF_SLICE_CHICKEN3,
+                   _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
+        I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_BWGTLB_DISABLE));
+
+        I915_WRITE(_3D_CHICKEN3,
+                   _3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(2));
+
+        I915_WRITE(COMMON_SLICE_CHICKEN2,
+                   _MASKED_BIT_ENABLE(GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE));
+
+        I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
+                   _MASKED_BIT_ENABLE(GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE));
+
+        /* WaSwitchSolVfFArbitrationPriority */
+        I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
+
+        /* WaPsrDPAMaskVBlankInSRD */
+        I915_WRITE(CHICKEN_PAR1_1,
+                   I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
+
+        /* WaPsrDPRSUnmaskVBlankInSRD */
+        for_each_pipe(i) {
+                I915_WRITE(CHICKEN_PIPESL_1(i),
+                           I915_READ(CHICKEN_PIPESL_1(i) |
+                                     DPRS_MASK_VBLANK_SRD));
+        }
+}
+
 static void haswell_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -5476,7 +5613,9 @@ static bool is_always_on_power_domain(struct drm_device *dev,
 
         BUG_ON(BIT(domain) & ~POWER_DOMAIN_MASK);
 
-        if (IS_HASWELL(dev)) {
+        if (IS_BROADWELL(dev)) {
+                always_on_domains = BDW_ALWAYS_ON_POWER_DOMAINS;
+        } else if (IS_HASWELL(dev)) {
                 always_on_domains = HSW_ALWAYS_ON_POWER_DOMAINS;
         } else {
                 WARN_ON(1);
@@ -5510,6 +5649,7 @@ static void __intel_set_power_well(struct drm_device *dev, bool enable)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         bool is_enabled, enable_requested;
+        unsigned long irqflags;
         uint32_t tmp;
 
         tmp = I915_READ(HSW_PWR_WELL_DRIVER);
@@ -5527,9 +5667,24 @@ static void __intel_set_power_well(struct drm_device *dev, bool enable)
                                       HSW_PWR_WELL_STATE_ENABLED), 20))
                                 DRM_ERROR("Timeout enabling power well\n");
                 }
+
+                if (IS_BROADWELL(dev)) {
+                        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+                        I915_WRITE(GEN8_DE_PIPE_IMR(PIPE_B),
+                                   dev_priv->de_irq_mask[PIPE_B]);
+                        I915_WRITE(GEN8_DE_PIPE_IER(PIPE_B),
+                                   ~dev_priv->de_irq_mask[PIPE_B] |
+                                   GEN8_PIPE_VBLANK);
+                        I915_WRITE(GEN8_DE_PIPE_IMR(PIPE_C),
+                                   dev_priv->de_irq_mask[PIPE_C]);
+                        I915_WRITE(GEN8_DE_PIPE_IER(PIPE_C),
+                                   ~dev_priv->de_irq_mask[PIPE_C] |
+                                   GEN8_PIPE_VBLANK);
+                        POSTING_READ(GEN8_DE_PIPE_IER(PIPE_C));
+                        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+                }
         } else {
                 if (enable_requested) {
-                        unsigned long irqflags;
                         enum pipe p;
 
                         I915_WRITE(HSW_PWR_WELL_DRIVER, 0);
@@ -5798,6 +5953,8 @@ void intel_init_pm(struct drm_device *dev)
                                 dev_priv->display.update_wm = NULL;
                         }
                         dev_priv->display.init_clock_gating = haswell_init_clock_gating;
+                } else if (INTEL_INFO(dev)->gen == 8) {
+                        dev_priv->display.init_clock_gating = gen8_init_clock_gating;
                 } else
                         dev_priv->display.update_wm = NULL;
         } else if (IS_VALLEYVIEW(dev)) {
@@ -5949,4 +6106,3 @@ void intel_pm_init(struct drm_device *dev)
         INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
                           intel_gen6_powersave_work);
 }
-
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 2dec134f75eb..c2f09d456300 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -360,6 +360,47 @@ gen7_render_ring_flush(struct intel_ring_buffer *ring,
         return 0;
 }
 
+static int
+gen8_render_ring_flush(struct intel_ring_buffer *ring,
+                       u32 invalidate_domains, u32 flush_domains)
+{
+        u32 flags = 0;
+        u32 scratch_addr = ring->scratch.gtt_offset + 128;
+        int ret;
+
+        flags |= PIPE_CONTROL_CS_STALL;
+
+        if (flush_domains) {
+                flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
+                flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
+        }
+        if (invalidate_domains) {
+                flags |= PIPE_CONTROL_TLB_INVALIDATE;
+                flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
+                flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
+                flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
+                flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
+                flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
+                flags |= PIPE_CONTROL_QW_WRITE;
+                flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
+        }
+
+        ret = intel_ring_begin(ring, 6);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring, GFX_OP_PIPE_CONTROL(6));
+        intel_ring_emit(ring, flags);
+        intel_ring_emit(ring, scratch_addr);
+        intel_ring_emit(ring, 0);
+        intel_ring_emit(ring, 0);
+        intel_ring_emit(ring, 0);
+        intel_ring_advance(ring);
+
+        return 0;
+
+}
+
 static void ring_write_tail(struct intel_ring_buffer *ring,
                             u32 value)
 {
@@ -924,6 +965,7 @@ void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
         } else if (IS_GEN6(ring->dev)) {
                 mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
         } else {
+                /* XXX: gen8 returns to sanity */
                 mmio = RING_HWS_PGA(ring->mmio_base);
         }
 
@@ -1066,6 +1108,52 @@ hsw_vebox_put_irq(struct intel_ring_buffer *ring)
         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 }
 
+static bool
+gen8_ring_get_irq(struct intel_ring_buffer *ring)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long flags;
+
+        if (!dev->irq_enabled)
+                return false;
+
+        spin_lock_irqsave(&dev_priv->irq_lock, flags);
+        if (ring->irq_refcount++ == 0) {
+                if (HAS_L3_DPF(dev) && ring->id == RCS) {
+                        I915_WRITE_IMR(ring,
+                                       ~(ring->irq_enable_mask |
+                                         GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
+                } else {
+                        I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
+                }
+                POSTING_READ(RING_IMR(ring->mmio_base));
+        }
+        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+
+        return true;
+}
+
+static void
+gen8_ring_put_irq(struct intel_ring_buffer *ring)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev_priv->irq_lock, flags);
+        if (--ring->irq_refcount == 0) {
+                if (HAS_L3_DPF(dev) && ring->id == RCS) {
+                        I915_WRITE_IMR(ring,
+                                       ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
+                } else {
+                        I915_WRITE_IMR(ring, ~0);
+                }
+                POSTING_READ(RING_IMR(ring->mmio_base));
+        }
+        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+}
+
 static int
 i965_dispatch_execbuffer(struct intel_ring_buffer *ring,
                          u32 offset, u32 length,
@@ -1624,6 +1712,8 @@ static int gen6_bsd_ring_flush(struct intel_ring_buffer *ring,
                 return ret;
 
         cmd = MI_FLUSH_DW;
+        if (INTEL_INFO(ring->dev)->gen >= 8)
+                cmd += 1;
         /*
          * Bspec vol 1c.5 - video engine command streamer:
          * "If ENABLED, all TLBs will be invalidated once the flush
@@ -1635,9 +1725,38 @@ static int gen6_bsd_ring_flush(struct intel_ring_buffer *ring,
                         MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
         intel_ring_emit(ring, cmd);
         intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
+        if (INTEL_INFO(ring->dev)->gen >= 8) {
+                intel_ring_emit(ring, 0); /* upper addr */
+                intel_ring_emit(ring, 0); /* value */
+        } else  {
+                intel_ring_emit(ring, 0);
+                intel_ring_emit(ring, MI_NOOP);
+        }
+        intel_ring_advance(ring);
+        return 0;
+}
+
+static int
+gen8_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
+                              u32 offset, u32 len,
+                              unsigned flags)
+{
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
+        bool ppgtt = dev_priv->mm.aliasing_ppgtt != NULL &&
+                !(flags & I915_DISPATCH_SECURE);
+        int ret;
+
+        ret = intel_ring_begin(ring, 4);
+        if (ret)
+                return ret;
+
+        /* FIXME(BDW): Address space and security selectors. */
+        intel_ring_emit(ring, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8));
+        intel_ring_emit(ring, offset);
         intel_ring_emit(ring, 0);
         intel_ring_emit(ring, MI_NOOP);
         intel_ring_advance(ring);
+
         return 0;
 }
 
@@ -1697,6 +1816,8 @@ static int gen6_ring_flush(struct intel_ring_buffer *ring,
                 return ret;
 
         cmd = MI_FLUSH_DW;
+        if (INTEL_INFO(ring->dev)->gen >= 8)
+                cmd += 1;
         /*
          * Bspec vol 1c.3 - blitter engine command streamer:
          * "If ENABLED, all TLBs will be invalidated once the flush
@@ -1708,8 +1829,13 @@ static int gen6_ring_flush(struct intel_ring_buffer *ring,
                         MI_FLUSH_DW_OP_STOREDW;
         intel_ring_emit(ring, cmd);
         intel_ring_emit(ring, I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
-        intel_ring_emit(ring, 0);
-        intel_ring_emit(ring, MI_NOOP);
+        if (INTEL_INFO(ring->dev)->gen >= 8) {
+                intel_ring_emit(ring, 0); /* upper addr */
+                intel_ring_emit(ring, 0); /* value */
+        } else  {
+                intel_ring_emit(ring, 0);
+                intel_ring_emit(ring, MI_NOOP);
+        }
         intel_ring_advance(ring);
 
         if (IS_GEN7(dev) && flush)
@@ -1732,8 +1858,14 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
                 ring->flush = gen7_render_ring_flush;
                 if (INTEL_INFO(dev)->gen == 6)
                         ring->flush = gen6_render_ring_flush;
-                ring->irq_get = gen6_ring_get_irq;
-                ring->irq_put = gen6_ring_put_irq;
+                if (INTEL_INFO(dev)->gen >= 8) {
+                        ring->flush = gen8_render_ring_flush;
+                        ring->irq_get = gen8_ring_get_irq;
+                        ring->irq_put = gen8_ring_put_irq;
+                } else {
+                        ring->irq_get = gen6_ring_get_irq;
+                        ring->irq_put = gen6_ring_put_irq;
+                }
                 ring->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
                 ring->get_seqno = gen6_ring_get_seqno;
                 ring->set_seqno = ring_set_seqno;
@@ -1775,6 +1907,8 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
         ring->write_tail = ring_write_tail;
         if (IS_HASWELL(dev))
                 ring->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
+        else if (IS_GEN8(dev))
+                ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
         else if (INTEL_INFO(dev)->gen >= 6)
                 ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
         else if (INTEL_INFO(dev)->gen >= 4)
@@ -1888,7 +2022,7 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev)
         ring->id = VCS;
 
         ring->write_tail = ring_write_tail;
-        if (IS_GEN6(dev) || IS_GEN7(dev)) {
+        if (INTEL_INFO(dev)->gen >= 6) {
                 ring->mmio_base = GEN6_BSD_RING_BASE;
                 /* gen6 bsd needs a special wa for tail updates */
                 if (IS_GEN6(dev))
@@ -1897,10 +2031,20 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev)
                 ring->add_request = gen6_add_request;
                 ring->get_seqno = gen6_ring_get_seqno;
                 ring->set_seqno = ring_set_seqno;
-                ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
-                ring->irq_get = gen6_ring_get_irq;
-                ring->irq_put = gen6_ring_put_irq;
-                ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
+                if (INTEL_INFO(dev)->gen >= 8) {
+                        ring->irq_enable_mask =
+                                GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
+                        ring->irq_get = gen8_ring_get_irq;
+                        ring->irq_put = gen8_ring_put_irq;
+                        ring->dispatch_execbuffer =
+                                gen8_ring_dispatch_execbuffer;
+                } else {
+                        ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
+                        ring->irq_get = gen6_ring_get_irq;
+                        ring->irq_put = gen6_ring_put_irq;
+                        ring->dispatch_execbuffer =
+                                gen6_ring_dispatch_execbuffer;
+                }
                 ring->sync_to = gen6_ring_sync;
                 ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VR;
                 ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_INVALID;
@@ -1946,10 +2090,18 @@ int intel_init_blt_ring_buffer(struct drm_device *dev)
         ring->add_request = gen6_add_request;
         ring->get_seqno = gen6_ring_get_seqno;
         ring->set_seqno = ring_set_seqno;
-        ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
-        ring->irq_get = gen6_ring_get_irq;
-        ring->irq_put = gen6_ring_put_irq;
-        ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
+        if (INTEL_INFO(dev)->gen >= 8) {
+                ring->irq_enable_mask =
+                        GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
+                ring->irq_get = gen8_ring_get_irq;
+                ring->irq_put = gen8_ring_put_irq;
+                ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
+        } else {
+                ring->irq_enable_mask = GT_BLT_USER_INTERRUPT;
+                ring->irq_get = gen6_ring_get_irq;
+                ring->irq_put = gen6_ring_put_irq;
+                ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
+        }
         ring->sync_to = gen6_ring_sync;
         ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_BR;
         ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_BV;
@@ -1978,10 +2130,19 @@ int intel_init_vebox_ring_buffer(struct drm_device *dev)
         ring->add_request = gen6_add_request;
         ring->get_seqno = gen6_ring_get_seqno;
         ring->set_seqno = ring_set_seqno;
-        ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
-        ring->irq_get = hsw_vebox_get_irq;
-        ring->irq_put = hsw_vebox_put_irq;
-        ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
+
+        if (INTEL_INFO(dev)->gen >= 8) {
+                ring->irq_enable_mask =
+                        GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
+                ring->irq_get = gen8_ring_get_irq;
+                ring->irq_put = gen8_ring_put_irq;
+                ring->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
+        } else {
+                ring->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
+                ring->irq_get = hsw_vebox_get_irq;
+                ring->irq_put = hsw_vebox_put_irq;
+                ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
+        }
         ring->sync_to = gen6_ring_sync;
         ring->semaphore_register[RCS] = MI_SEMAPHORE_SYNC_VER;
         ring->semaphore_register[VCS] = MI_SEMAPHORE_SYNC_VEV;
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 8afaad6bcc48..b9fabf826f7d 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -260,14 +260,14 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
         if (obj->tiling_mode != I915_TILING_NONE)
                 sprctl |= SPRITE_TILED;
 
-        if (IS_HASWELL(dev))
+        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
                 sprctl &= ~SPRITE_TRICKLE_FEED_DISABLE;
         else
                 sprctl |= SPRITE_TRICKLE_FEED_DISABLE;
 
         sprctl |= SPRITE_ENABLE;
 
-        if (IS_HASWELL(dev))
+        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
                 sprctl |= SPRITE_PIPE_CSC_ENABLE;
 
         intel_update_sprite_watermarks(plane, crtc, src_w, pixel_size, true,
@@ -306,7 +306,7 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
 
         /* HSW consolidates SPRTILEOFF and SPRLINOFF into a single SPROFFSET
          * register */
-        if (IS_HASWELL(dev))
+        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
                 I915_WRITE(SPROFFSET(pipe), (y << 16) | x);
         else if (obj->tiling_mode != I915_TILING_NONE)
                 I915_WRITE(SPRTILEOFF(pipe), (y << 16) | x);
@@ -955,7 +955,7 @@ int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
 
         obj = drm_mode_object_find(dev, set->plane_id, DRM_MODE_OBJECT_PLANE);
         if (!obj) {
-                ret = -EINVAL;
+                ret = -ENOENT;
                 goto out_unlock;
         }
 
@@ -984,7 +984,7 @@ int intel_sprite_get_colorkey(struct drm_device *dev, void *data,
 
         obj = drm_mode_object_find(dev, get->plane_id, DRM_MODE_OBJECT_PLANE);
         if (!obj) {
-                ret = -EINVAL;
+                ret = -ENOENT;
                 goto out_unlock;
         }
 
@@ -1092,6 +1092,7 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
                 break;
 
         case 7:
+        case 8:
                 if (IS_IVYBRIDGE(dev)) {
                         intel_plane->can_scale = true;
                         intel_plane->max_downscale = 2;
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index a881906969eb..5103d80fc3ae 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -93,7 +93,7 @@ static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
 {
         u32 forcewake_ack;
 
-        if (IS_HASWELL(dev_priv->dev))
+        if (IS_HASWELL(dev_priv->dev) || IS_GEN8(dev_priv->dev))
                 forcewake_ack = FORCEWAKE_ACK_HSW;
         else
                 forcewake_ack = FORCEWAKE_MT_ACK;
@@ -112,7 +112,8 @@ static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
                 DRM_ERROR("Timed out waiting for forcewake to ack request.\n");
 
         /* WaRsForcewakeWaitTC0:ivb,hsw */
-        __gen6_gt_wait_for_thread_c0(dev_priv);
+        if (INTEL_INFO(dev_priv->dev)->gen < 8)
+                __gen6_gt_wait_for_thread_c0(dev_priv);
 }
 
 static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv)
@@ -459,6 +460,46 @@ hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace)
         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
 }
 
+static const u32 gen8_shadowed_regs[] = {
+        FORCEWAKE_MT,
+        GEN6_RPNSWREQ,
+        GEN6_RC_VIDEO_FREQ,
+        RING_TAIL(RENDER_RING_BASE),
+        RING_TAIL(GEN6_BSD_RING_BASE),
+        RING_TAIL(VEBOX_RING_BASE),
+        RING_TAIL(BLT_RING_BASE),
+        /* TODO: Other registers are not yet used */
+};
+
+static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg)
+{
+        int i;
+        for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++)
+                if (reg == gen8_shadowed_regs[i])
+                        return true;
+
+        return false;
+}
+
+#define __gen8_write(x) \
+static void \
+gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
+        bool __needs_put = !is_gen8_shadowed(dev_priv, reg); \
+        REG_WRITE_HEADER; \
+        if (__needs_put) { \
+                dev_priv->uncore.funcs.force_wake_get(dev_priv); \
+        } \
+        __raw_i915_write##x(dev_priv, reg, val); \
+        if (__needs_put) { \
+                dev_priv->uncore.funcs.force_wake_put(dev_priv); \
+        } \
+        spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
+}
+
+__gen8_write(8)
+__gen8_write(16)
+__gen8_write(32)
+__gen8_write(64)
 __hsw_write(8)
 __hsw_write(16)
 __hsw_write(32)
@@ -476,6 +517,7 @@ __gen4_write(16)
 __gen4_write(32)
 __gen4_write(64)
 
+#undef __gen8_write
 #undef __hsw_write
 #undef __gen6_write
 #undef __gen5_write
@@ -492,7 +534,7 @@ void intel_uncore_init(struct drm_device *dev)
         if (IS_VALLEYVIEW(dev)) {
                 dev_priv->uncore.funcs.force_wake_get = vlv_force_wake_get;
                 dev_priv->uncore.funcs.force_wake_put = vlv_force_wake_put;
-        } else if (IS_HASWELL(dev)) {
+        } else if (IS_HASWELL(dev) || IS_GEN8(dev)) {
                 dev_priv->uncore.funcs.force_wake_get = __gen6_gt_force_wake_mt_get;
                 dev_priv->uncore.funcs.force_wake_put = __gen6_gt_force_wake_mt_put;
         } else if (IS_IVYBRIDGE(dev)) {
@@ -534,6 +576,16 @@ void intel_uncore_init(struct drm_device *dev)
         }
 
         switch (INTEL_INFO(dev)->gen) {
+        default:
+                dev_priv->uncore.funcs.mmio_writeb  = gen8_write8;
+                dev_priv->uncore.funcs.mmio_writew  = gen8_write16;
+                dev_priv->uncore.funcs.mmio_writel  = gen8_write32;
+                dev_priv->uncore.funcs.mmio_writeq  = gen8_write64;
+                dev_priv->uncore.funcs.mmio_readb  = gen6_read8;
+                dev_priv->uncore.funcs.mmio_readw  = gen6_read16;
+                dev_priv->uncore.funcs.mmio_readl  = gen6_read32;
+                dev_priv->uncore.funcs.mmio_readq  = gen6_read64;
+                break;
         case 7:
         case 6:
                 if (IS_HASWELL(dev)) {
@@ -767,6 +819,7 @@ static int gen6_do_reset(struct drm_device *dev)
 int intel_gpu_reset(struct drm_device *dev)
 {
         switch (INTEL_INFO(dev)->gen) {
+        case 8:
         case 7:
         case 6: return gen6_do_reset(dev);
         case 5: return ironlake_do_reset(dev);
diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
index 503a414cbdad..ee6ed633b7b1 100644
--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
+++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
@@ -765,8 +765,6 @@ static int mga_crtc_do_set_base(struct drm_crtc *crtc,
         }
         mgag200_bo_unreserve(bo);
 
-        DRM_INFO("mga base %llx\n", gpu_addr);
-
         mga_set_start_address(crtc, (u32)gpu_addr);
 
         return 0;
diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
index e17914889e54..e5fa12b0d21e 100644
--- a/drivers/gpu/drm/msm/Makefile
+++ b/drivers/gpu/drm/msm/Makefile
@@ -21,6 +21,7 @@ msm-y := \
         msm_drv.o \
         msm_fb.o \
         msm_gem.o \
+        msm_gem_prime.o \
         msm_gem_submit.o \
         msm_gpu.o \
         msm_ringbuffer.o
diff --git a/drivers/gpu/drm/msm/adreno/a2xx.xml.h b/drivers/gpu/drm/msm/adreno/a2xx.xml.h
index 35463864b959..9588098741b5 100644
--- a/drivers/gpu/drm/msm/adreno/a2xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a2xx.xml.h
@@ -4,16 +4,16 @@
 /* Autogenerated file, DO NOT EDIT manually!
 
 This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://0x04.net/cgit/index.cgi/rules-ng-ng
-git clone git://0x04.net/rules-ng-ng
+http://github.com/freedreno/envytools/
+git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/adreno.xml              (    327 bytes, from 2013-07-05 19:21:12)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1453 bytes, from 2013-03-31 16:51:27)
-- /home/robclark/src/freedreno/envytools/rnndb/a2xx/a2xx.xml           (  30005 bytes, from 2013-07-19 21:30:48)
+- /home/robclark/src/freedreno/envytools/rnndb/a2xx/a2xx.xml           (  31003 bytes, from 2013-09-19 18:50:16)
 - /home/robclark/src/freedreno/envytools/rnndb/adreno_common.xml       (   8983 bytes, from 2013-07-24 01:38:36)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno_pm4.xml          (   9712 bytes, from 2013-05-26 15:22:37)
-- /home/robclark/src/freedreno/envytools/rnndb/a3xx/a3xx.xml           (  51415 bytes, from 2013-08-03 14:26:05)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno_pm4.xml          (   9759 bytes, from 2013-09-10 00:52:33)
+- /home/robclark/src/freedreno/envytools/rnndb/a3xx/a3xx.xml           (  51983 bytes, from 2013-09-10 00:52:32)
 
 Copyright (C) 2013 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
@@ -317,6 +317,38 @@ static inline uint32_t A2XX_RBBM_STATUS_CMDFIFO_AVAIL(uint32_t val)
 #define A2XX_RBBM_STATUS_RB_CNTX_BUSY                           0x40000000
 #define A2XX_RBBM_STATUS_GUI_ACTIVE                             0x80000000
 
+#define REG_A2XX_MH_ARBITER_CONFIG                              0x00000a40
+#define A2XX_MH_ARBITER_CONFIG_SAME_PAGE_LIMIT__MASK            0x0000003f
+#define A2XX_MH_ARBITER_CONFIG_SAME_PAGE_LIMIT__SHIFT           0
+static inline uint32_t A2XX_MH_ARBITER_CONFIG_SAME_PAGE_LIMIT(uint32_t val)
+{
+        return ((val) << A2XX_MH_ARBITER_CONFIG_SAME_PAGE_LIMIT__SHIFT) & A2XX_MH_ARBITER_CONFIG_SAME_PAGE_LIMIT__MASK;
+}
+#define A2XX_MH_ARBITER_CONFIG_SAME_PAGE_GRANULARITY            0x00000040
+#define A2XX_MH_ARBITER_CONFIG_L1_ARB_ENABLE                    0x00000080
+#define A2XX_MH_ARBITER_CONFIG_L1_ARB_HOLD_ENABLE               0x00000100
+#define A2XX_MH_ARBITER_CONFIG_L2_ARB_CONTROL                   0x00000200
+#define A2XX_MH_ARBITER_CONFIG_PAGE_SIZE__MASK                  0x00001c00
+#define A2XX_MH_ARBITER_CONFIG_PAGE_SIZE__SHIFT                 10
+static inline uint32_t A2XX_MH_ARBITER_CONFIG_PAGE_SIZE(uint32_t val)
+{
+        return ((val) << A2XX_MH_ARBITER_CONFIG_PAGE_SIZE__SHIFT) & A2XX_MH_ARBITER_CONFIG_PAGE_SIZE__MASK;
+}
+#define A2XX_MH_ARBITER_CONFIG_TC_REORDER_ENABLE                0x00002000
+#define A2XX_MH_ARBITER_CONFIG_TC_ARB_HOLD_ENABLE               0x00004000
+#define A2XX_MH_ARBITER_CONFIG_IN_FLIGHT_LIMIT_ENABLE           0x00008000
+#define A2XX_MH_ARBITER_CONFIG_IN_FLIGHT_LIMIT__MASK            0x003f0000
+#define A2XX_MH_ARBITER_CONFIG_IN_FLIGHT_LIMIT__SHIFT           16
+static inline uint32_t A2XX_MH_ARBITER_CONFIG_IN_FLIGHT_LIMIT(uint32_t val)
+{
+        return ((val) << A2XX_MH_ARBITER_CONFIG_IN_FLIGHT_LIMIT__SHIFT) & A2XX_MH_ARBITER_CONFIG_IN_FLIGHT_LIMIT__MASK;
+}
+#define A2XX_MH_ARBITER_CONFIG_CP_CLNT_ENABLE                   0x00400000
+#define A2XX_MH_ARBITER_CONFIG_VGT_CLNT_ENABLE                  0x00800000
+#define A2XX_MH_ARBITER_CONFIG_TC_CLNT_ENABLE                   0x01000000
+#define A2XX_MH_ARBITER_CONFIG_RB_CLNT_ENABLE                   0x02000000
+#define A2XX_MH_ARBITER_CONFIG_PA_CLNT_ENABLE                   0x04000000
+
 #define REG_A2XX_A220_VSC_BIN_SIZE                              0x00000c01
 #define A2XX_A220_VSC_BIN_SIZE_WIDTH__MASK                      0x0000001f
 #define A2XX_A220_VSC_BIN_SIZE_WIDTH__SHIFT                     0
diff --git a/drivers/gpu/drm/msm/adreno/a3xx.xml.h b/drivers/gpu/drm/msm/adreno/a3xx.xml.h
index d183516067b4..d4afdf657559 100644
--- a/drivers/gpu/drm/msm/adreno/a3xx.xml.h
+++ b/drivers/gpu/drm/msm/adreno/a3xx.xml.h
@@ -4,16 +4,16 @@
 /* Autogenerated file, DO NOT EDIT manually!
 
 This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://0x04.net/cgit/index.cgi/rules-ng-ng
-git clone git://0x04.net/rules-ng-ng
+http://github.com/freedreno/envytools/
+git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/adreno.xml              (    327 bytes, from 2013-07-05 19:21:12)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1453 bytes, from 2013-03-31 16:51:27)
-- /home/robclark/src/freedreno/envytools/rnndb/a2xx/a2xx.xml           (  30005 bytes, from 2013-07-19 21:30:48)
+- /home/robclark/src/freedreno/envytools/rnndb/a2xx/a2xx.xml           (  31003 bytes, from 2013-09-19 18:50:16)
 - /home/robclark/src/freedreno/envytools/rnndb/adreno_common.xml       (   8983 bytes, from 2013-07-24 01:38:36)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno_pm4.xml          (   9712 bytes, from 2013-05-26 15:22:37)
-- /home/robclark/src/freedreno/envytools/rnndb/a3xx/a3xx.xml           (  51415 bytes, from 2013-08-03 14:26:05)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno_pm4.xml          (   9759 bytes, from 2013-09-10 00:52:33)
+- /home/robclark/src/freedreno/envytools/rnndb/a3xx/a3xx.xml           (  51983 bytes, from 2013-09-10 00:52:32)
 
 Copyright (C) 2013 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
@@ -637,11 +637,12 @@ static inline uint32_t A3XX_GRAS_SU_POLY_OFFSET_OFFSET(float val)
 #define REG_A3XX_GRAS_SU_MODE_CONTROL                           0x00002070
 #define A3XX_GRAS_SU_MODE_CONTROL_CULL_FRONT                    0x00000001
 #define A3XX_GRAS_SU_MODE_CONTROL_CULL_BACK                     0x00000002
-#define A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__MASK           0x000007fc
-#define A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__SHIFT          2
-static inline uint32_t A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH(uint32_t val)
+#define A3XX_GRAS_SU_MODE_CONTROL_FRONT_CW                      0x00000004
+#define A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__MASK           0x000007f8
+#define A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__SHIFT          3
+static inline uint32_t A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH(float val)
 {
-        return ((val) << A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__SHIFT) & A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__MASK;
+        return ((((uint32_t)(val * 4.0))) << A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__SHIFT) & A3XX_GRAS_SU_MODE_CONTROL_LINEHALFWIDTH__MASK;
 }
 #define A3XX_GRAS_SU_MODE_CONTROL_POLY_OFFSET                   0x00000800
 
@@ -745,6 +746,7 @@ static inline uint32_t A3XX_RB_RENDER_CONTROL_BIN_WIDTH(uint32_t val)
 }
 #define A3XX_RB_RENDER_CONTROL_DISABLE_COLOR_PIPE               0x00001000
 #define A3XX_RB_RENDER_CONTROL_ENABLE_GMEM                      0x00002000
+#define A3XX_RB_RENDER_CONTROL_ALPHA_TEST                       0x00400000
 #define A3XX_RB_RENDER_CONTROL_ALPHA_TEST_FUNC__MASK            0x07000000
 #define A3XX_RB_RENDER_CONTROL_ALPHA_TEST_FUNC__SHIFT           24
 static inline uint32_t A3XX_RB_RENDER_CONTROL_ALPHA_TEST_FUNC(enum adreno_compare_func val)
@@ -767,7 +769,19 @@ static inline uint32_t A3XX_RB_MSAA_CONTROL_SAMPLE_MASK(uint32_t val)
         return ((val) << A3XX_RB_MSAA_CONTROL_SAMPLE_MASK__SHIFT) & A3XX_RB_MSAA_CONTROL_SAMPLE_MASK__MASK;
 }
 
-#define REG_A3XX_UNKNOWN_20C3                                   0x000020c3
+#define REG_A3XX_RB_ALPHA_REF                                   0x000020c3
+#define A3XX_RB_ALPHA_REF_UINT__MASK                            0x0000ff00
+#define A3XX_RB_ALPHA_REF_UINT__SHIFT                           8
+static inline uint32_t A3XX_RB_ALPHA_REF_UINT(uint32_t val)
+{
+        return ((val) << A3XX_RB_ALPHA_REF_UINT__SHIFT) & A3XX_RB_ALPHA_REF_UINT__MASK;
+}
+#define A3XX_RB_ALPHA_REF_FLOAT__MASK                           0xffff0000
+#define A3XX_RB_ALPHA_REF_FLOAT__SHIFT                          16
+static inline uint32_t A3XX_RB_ALPHA_REF_FLOAT(float val)
+{
+        return ((util_float_to_half(val)) << A3XX_RB_ALPHA_REF_FLOAT__SHIFT) & A3XX_RB_ALPHA_REF_FLOAT__MASK;
+}
 
 static inline uint32_t REG_A3XX_RB_MRT(uint32_t i0) { return 0x000020c4 + 0x4*i0; }
 
@@ -1002,7 +1016,7 @@ static inline uint32_t A3XX_RB_COPY_DEST_INFO_ENDIAN(enum adreno_rb_surface_endi
 #define REG_A3XX_RB_DEPTH_CONTROL                               0x00002100
 #define A3XX_RB_DEPTH_CONTROL_Z_ENABLE                          0x00000002
 #define A3XX_RB_DEPTH_CONTROL_Z_WRITE_ENABLE                    0x00000004
-#define A3XX_RB_DEPTH_CONTROL_EARLY_Z_ENABLE                    0x00000008
+#define A3XX_RB_DEPTH_CONTROL_EARLY_Z_DISABLE                   0x00000008
 #define A3XX_RB_DEPTH_CONTROL_ZFUNC__MASK                       0x00000070
 #define A3XX_RB_DEPTH_CONTROL_ZFUNC__SHIFT                      4
 static inline uint32_t A3XX_RB_DEPTH_CONTROL_ZFUNC(enum adreno_compare_func val)
@@ -1038,7 +1052,8 @@ static inline uint32_t A3XX_RB_DEPTH_PITCH(uint32_t val)
 
 #define REG_A3XX_RB_STENCIL_CONTROL                             0x00002104
 #define A3XX_RB_STENCIL_CONTROL_STENCIL_ENABLE                  0x00000001
-#define A3XX_RB_STENCIL_CONTROL_STENCIL_ENABLE_BF               0x00000004
+#define A3XX_RB_STENCIL_CONTROL_STENCIL_ENABLE_BF               0x00000002
+#define A3XX_RB_STENCIL_CONTROL_STENCIL_READ                    0x00000004
 #define A3XX_RB_STENCIL_CONTROL_FUNC__MASK                      0x00000700
 #define A3XX_RB_STENCIL_CONTROL_FUNC__SHIFT                     8
 static inline uint32_t A3XX_RB_STENCIL_CONTROL_FUNC(enum adreno_compare_func val)
@@ -2074,6 +2089,7 @@ static inline uint32_t A3XX_UCHE_CACHE_INVALIDATE1_REG_OPCODE(enum a3xx_cache_op
 #define REG_A3XX_TP_PERFCOUNTER5_SELECT                         0x00000f09
 
 #define REG_A3XX_TEX_SAMP_0                                     0x00000000
+#define A3XX_TEX_SAMP_0_MIPFILTER_LINEAR                        0x00000002
 #define A3XX_TEX_SAMP_0_XY_MAG__MASK                            0x0000000c
 #define A3XX_TEX_SAMP_0_XY_MAG__SHIFT                           2
 static inline uint32_t A3XX_TEX_SAMP_0_XY_MAG(enum a3xx_tex_filter val)
@@ -2134,6 +2150,12 @@ static inline uint32_t A3XX_TEX_CONST_0_SWIZ_W(enum a3xx_tex_swiz val)
 {
         return ((val) << A3XX_TEX_CONST_0_SWIZ_W__SHIFT) & A3XX_TEX_CONST_0_SWIZ_W__MASK;
 }
+#define A3XX_TEX_CONST_0_MIPLVLS__MASK                          0x000f0000
+#define A3XX_TEX_CONST_0_MIPLVLS__SHIFT                         16
+static inline uint32_t A3XX_TEX_CONST_0_MIPLVLS(uint32_t val)
+{
+        return ((val) << A3XX_TEX_CONST_0_MIPLVLS__SHIFT) & A3XX_TEX_CONST_0_MIPLVLS__MASK;
+}
 #define A3XX_TEX_CONST_0_FMT__MASK                              0x1fc00000
 #define A3XX_TEX_CONST_0_FMT__SHIFT                             22
 static inline uint32_t A3XX_TEX_CONST_0_FMT(enum a3xx_tex_fmt val)
diff --git a/drivers/gpu/drm/msm/adreno/adreno_common.xml.h b/drivers/gpu/drm/msm/adreno/adreno_common.xml.h
index 61979d458ac0..33dcc606c7c5 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_common.xml.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_common.xml.h
@@ -4,16 +4,16 @@
 /* Autogenerated file, DO NOT EDIT manually!
 
 This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://0x04.net/cgit/index.cgi/rules-ng-ng
-git clone git://0x04.net/rules-ng-ng
+http://github.com/freedreno/envytools/
+git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/adreno.xml              (    327 bytes, from 2013-07-05 19:21:12)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1453 bytes, from 2013-03-31 16:51:27)
-- /home/robclark/src/freedreno/envytools/rnndb/a2xx/a2xx.xml           (  30005 bytes, from 2013-07-19 21:30:48)
+- /home/robclark/src/freedreno/envytools/rnndb/a2xx/a2xx.xml           (  31003 bytes, from 2013-09-19 18:50:16)
 - /home/robclark/src/freedreno/envytools/rnndb/adreno_common.xml       (   8983 bytes, from 2013-07-24 01:38:36)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno_pm4.xml          (   9712 bytes, from 2013-05-26 15:22:37)
-- /home/robclark/src/freedreno/envytools/rnndb/a3xx/a3xx.xml           (  51415 bytes, from 2013-08-03 14:26:05)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno_pm4.xml          (   9759 bytes, from 2013-09-10 00:52:33)
+- /home/robclark/src/freedreno/envytools/rnndb/a3xx/a3xx.xml           (  51983 bytes, from 2013-09-10 00:52:32)
 
 Copyright (C) 2013 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
diff --git a/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h b/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
index 94c13f418e75..259ad709b0cc 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h
@@ -4,16 +4,16 @@
 /* Autogenerated file, DO NOT EDIT manually!
 
 This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://0x04.net/cgit/index.cgi/rules-ng-ng
-git clone git://0x04.net/rules-ng-ng
+http://github.com/freedreno/envytools/
+git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/adreno.xml              (    327 bytes, from 2013-07-05 19:21:12)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1453 bytes, from 2013-03-31 16:51:27)
-- /home/robclark/src/freedreno/envytools/rnndb/a2xx/a2xx.xml           (  30005 bytes, from 2013-07-19 21:30:48)
+- /home/robclark/src/freedreno/envytools/rnndb/a2xx/a2xx.xml           (  31003 bytes, from 2013-09-19 18:50:16)
 - /home/robclark/src/freedreno/envytools/rnndb/adreno_common.xml       (   8983 bytes, from 2013-07-24 01:38:36)
-- /home/robclark/src/freedreno/envytools/rnndb/adreno_pm4.xml          (   9712 bytes, from 2013-05-26 15:22:37)
-- /home/robclark/src/freedreno/envytools/rnndb/a3xx/a3xx.xml           (  51415 bytes, from 2013-08-03 14:26:05)
+- /home/robclark/src/freedreno/envytools/rnndb/adreno_pm4.xml          (   9759 bytes, from 2013-09-10 00:52:33)
+- /home/robclark/src/freedreno/envytools/rnndb/a3xx/a3xx.xml           (  51983 bytes, from 2013-09-10 00:52:32)
 
 Copyright (C) 2013 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
diff --git a/drivers/gpu/drm/msm/dsi/dsi.xml.h b/drivers/gpu/drm/msm/dsi/dsi.xml.h
index 6f8396be431d..6d4c62bf70dc 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.xml.h
+++ b/drivers/gpu/drm/msm/dsi/dsi.xml.h
@@ -4,13 +4,13 @@
 /* Autogenerated file, DO NOT EDIT manually!
 
 This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://0x04.net/cgit/index.cgi/rules-ng-ng
-git clone git://0x04.net/rules-ng-ng
+http://github.com/freedreno/envytools/
+git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    595 bytes, from 2013-07-05 19:21:12)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1453 bytes, from 2013-03-31 16:51:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp4/mdp4.xml           (  19332 bytes, from 2013-08-16 22:16:36)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp4/mdp4.xml           (  19332 bytes, from 2013-10-07 16:36:48)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  11712 bytes, from 2013-08-17 17:13:43)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    344 bytes, from 2013-08-11 19:26:32)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1544 bytes, from 2013-08-16 19:17:05)
diff --git a/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h b/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h
index aefc1b8feae9..d1df38bf5747 100644
--- a/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h
+++ b/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h
@@ -4,13 +4,13 @@
 /* Autogenerated file, DO NOT EDIT manually!
 
 This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://0x04.net/cgit/index.cgi/rules-ng-ng
-git clone git://0x04.net/rules-ng-ng
+http://github.com/freedreno/envytools/
+git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    595 bytes, from 2013-07-05 19:21:12)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1453 bytes, from 2013-03-31 16:51:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp4/mdp4.xml           (  19332 bytes, from 2013-08-16 22:16:36)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp4/mdp4.xml           (  19332 bytes, from 2013-10-07 16:36:48)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  11712 bytes, from 2013-08-17 17:13:43)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    344 bytes, from 2013-08-11 19:26:32)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1544 bytes, from 2013-08-16 19:17:05)
diff --git a/drivers/gpu/drm/msm/dsi/sfpb.xml.h b/drivers/gpu/drm/msm/dsi/sfpb.xml.h
index a225e8170b2a..0030a111302d 100644
--- a/drivers/gpu/drm/msm/dsi/sfpb.xml.h
+++ b/drivers/gpu/drm/msm/dsi/sfpb.xml.h
@@ -4,13 +4,13 @@
 /* Autogenerated file, DO NOT EDIT manually!
 
 This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://0x04.net/cgit/index.cgi/rules-ng-ng
-git clone git://0x04.net/rules-ng-ng
+http://github.com/freedreno/envytools/
+git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    595 bytes, from 2013-07-05 19:21:12)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1453 bytes, from 2013-03-31 16:51:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp4/mdp4.xml           (  19332 bytes, from 2013-08-16 22:16:36)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp4/mdp4.xml           (  19332 bytes, from 2013-10-07 16:36:48)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  11712 bytes, from 2013-08-17 17:13:43)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    344 bytes, from 2013-08-11 19:26:32)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1544 bytes, from 2013-08-16 19:17:05)
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.xml.h b/drivers/gpu/drm/msm/hdmi/hdmi.xml.h
index f5fa4865e059..4e939f82918c 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi.xml.h
+++ b/drivers/gpu/drm/msm/hdmi/hdmi.xml.h
@@ -4,13 +4,13 @@
 /* Autogenerated file, DO NOT EDIT manually!
 
 This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://0x04.net/cgit/index.cgi/rules-ng-ng
-git clone git://0x04.net/rules-ng-ng
+http://github.com/freedreno/envytools/
+git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    595 bytes, from 2013-07-05 19:21:12)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1453 bytes, from 2013-03-31 16:51:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp4/mdp4.xml           (  19332 bytes, from 2013-08-16 22:16:36)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp4/mdp4.xml           (  19332 bytes, from 2013-10-07 16:36:48)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  11712 bytes, from 2013-08-17 17:13:43)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    344 bytes, from 2013-08-11 19:26:32)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1544 bytes, from 2013-08-16 19:17:05)
diff --git a/drivers/gpu/drm/msm/hdmi/qfprom.xml.h b/drivers/gpu/drm/msm/hdmi/qfprom.xml.h
index bee36363bcd0..dbde4f6339b9 100644
--- a/drivers/gpu/drm/msm/hdmi/qfprom.xml.h
+++ b/drivers/gpu/drm/msm/hdmi/qfprom.xml.h
@@ -4,13 +4,13 @@
 /* Autogenerated file, DO NOT EDIT manually!
 
 This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://0x04.net/cgit/index.cgi/rules-ng-ng
-git clone git://0x04.net/rules-ng-ng
+http://github.com/freedreno/envytools/
+git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    595 bytes, from 2013-07-05 19:21:12)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1453 bytes, from 2013-03-31 16:51:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp4/mdp4.xml           (  19332 bytes, from 2013-08-16 22:16:36)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp4/mdp4.xml           (  19332 bytes, from 2013-10-07 16:36:48)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  11712 bytes, from 2013-08-17 17:13:43)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    344 bytes, from 2013-08-11 19:26:32)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1544 bytes, from 2013-08-16 19:17:05)
diff --git a/drivers/gpu/drm/msm/mdp4/mdp4.xml.h b/drivers/gpu/drm/msm/mdp4/mdp4.xml.h
index bbeeebe2db55..9908ffe1c3ad 100644
--- a/drivers/gpu/drm/msm/mdp4/mdp4.xml.h
+++ b/drivers/gpu/drm/msm/mdp4/mdp4.xml.h
@@ -4,13 +4,13 @@
 /* Autogenerated file, DO NOT EDIT manually!
 
 This file was generated by the rules-ng-ng headergen tool in this git repository:
-http://0x04.net/cgit/index.cgi/rules-ng-ng
-git clone git://0x04.net/rules-ng-ng
+http://github.com/freedreno/envytools/
+git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
 - /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    595 bytes, from 2013-07-05 19:21:12)
 - /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1453 bytes, from 2013-03-31 16:51:27)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp4/mdp4.xml           (  19332 bytes, from 2013-08-16 22:16:36)
+- /home/robclark/src/freedreno/envytools/rnndb/mdp4/mdp4.xml           (  19332 bytes, from 2013-10-07 16:36:48)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  11712 bytes, from 2013-08-17 17:13:43)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    344 bytes, from 2013-08-11 19:26:32)
 - /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1544 bytes, from 2013-08-16 19:17:05)
@@ -42,28 +42,28 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */
 
 
-enum mpd4_bpc {
+enum mdp4_bpc {
         BPC1 = 0,
         BPC5 = 1,
         BPC6 = 2,
         BPC8 = 3,
 };
 
-enum mpd4_bpc_alpha {
+enum mdp4_bpc_alpha {
         BPC1A = 0,
         BPC4A = 1,
         BPC6A = 2,
         BPC8A = 3,
 };
 
-enum mpd4_alpha_type {
+enum mdp4_alpha_type {
         FG_CONST = 0,
         BG_CONST = 1,
         FG_PIXEL = 2,
         BG_PIXEL = 3,
 };
 
-enum mpd4_pipe {
+enum mdp4_pipe {
         VG1 = 0,
         VG2 = 1,
         RGB1 = 2,
@@ -73,13 +73,13 @@ enum mpd4_pipe {
         VG4 = 6,
 };
 
-enum mpd4_mixer {
+enum mdp4_mixer {
         MIXER0 = 0,
         MIXER1 = 1,
         MIXER2 = 2,
 };
 
-enum mpd4_mixer_stage_id {
+enum mdp4_mixer_stage_id {
         STAGE_UNUSED = 0,
         STAGE_BASE = 1,
         STAGE0 = 2,
@@ -194,56 +194,56 @@ static inline uint32_t MDP4_DISP_INTF_SEL_EXT(enum mdp4_intf val)
 #define REG_MDP4_LAYERMIXER2_IN_CFG                             0x000100f0
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE0__MASK                     0x00000007
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE0__SHIFT                    0
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE0(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE0(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE0__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE0__MASK;
 }
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE0_MIXER1                    0x00000008
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE1__MASK                     0x00000070
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE1__SHIFT                    4
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE1(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE1(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE1__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE1__MASK;
 }
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE1_MIXER1                    0x00000080
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE2__MASK                     0x00000700
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE2__SHIFT                    8
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE2(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE2(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE2__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE2__MASK;
 }
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE2_MIXER1                    0x00000800
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE3__MASK                     0x00007000
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE3__SHIFT                    12
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE3(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE3(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE3__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE3__MASK;
 }
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE3_MIXER1                    0x00008000
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE4__MASK                     0x00070000
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE4__SHIFT                    16
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE4(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE4(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE4__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE4__MASK;
 }
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE4_MIXER1                    0x00080000
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE5__MASK                     0x00700000
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE5__SHIFT                    20
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE5(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE5(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE5__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE5__MASK;
 }
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE5_MIXER1                    0x00800000
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE6__MASK                     0x07000000
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE6__SHIFT                    24
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE6(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE6(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE6__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE6__MASK;
 }
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE6_MIXER1                    0x08000000
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE7__MASK                     0x70000000
 #define MDP4_LAYERMIXER2_IN_CFG_PIPE7__SHIFT                    28
-static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE7(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE7(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER2_IN_CFG_PIPE7__SHIFT) & MDP4_LAYERMIXER2_IN_CFG_PIPE7__MASK;
 }
@@ -254,56 +254,56 @@ static inline uint32_t MDP4_LAYERMIXER2_IN_CFG_PIPE7(enum mpd4_mixer_stage_id va
 #define REG_MDP4_LAYERMIXER_IN_CFG                              0x00010100
 #define MDP4_LAYERMIXER_IN_CFG_PIPE0__MASK                      0x00000007
 #define MDP4_LAYERMIXER_IN_CFG_PIPE0__SHIFT                     0
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE0(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE0(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE0__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE0__MASK;
 }
 #define MDP4_LAYERMIXER_IN_CFG_PIPE0_MIXER1                     0x00000008
 #define MDP4_LAYERMIXER_IN_CFG_PIPE1__MASK                      0x00000070
 #define MDP4_LAYERMIXER_IN_CFG_PIPE1__SHIFT                     4
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE1(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE1(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE1__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE1__MASK;
 }
 #define MDP4_LAYERMIXER_IN_CFG_PIPE1_MIXER1                     0x00000080
 #define MDP4_LAYERMIXER_IN_CFG_PIPE2__MASK                      0x00000700
 #define MDP4_LAYERMIXER_IN_CFG_PIPE2__SHIFT                     8
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE2(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE2(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE2__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE2__MASK;
 }
 #define MDP4_LAYERMIXER_IN_CFG_PIPE2_MIXER1                     0x00000800
 #define MDP4_LAYERMIXER_IN_CFG_PIPE3__MASK                      0x00007000
 #define MDP4_LAYERMIXER_IN_CFG_PIPE3__SHIFT                     12
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE3(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE3(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE3__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE3__MASK;
 }
 #define MDP4_LAYERMIXER_IN_CFG_PIPE3_MIXER1                     0x00008000
 #define MDP4_LAYERMIXER_IN_CFG_PIPE4__MASK                      0x00070000
 #define MDP4_LAYERMIXER_IN_CFG_PIPE4__SHIFT                     16
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE4(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE4(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE4__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE4__MASK;
 }
 #define MDP4_LAYERMIXER_IN_CFG_PIPE4_MIXER1                     0x00080000
 #define MDP4_LAYERMIXER_IN_CFG_PIPE5__MASK                      0x00700000
 #define MDP4_LAYERMIXER_IN_CFG_PIPE5__SHIFT                     20
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE5(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE5(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE5__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE5__MASK;
 }
 #define MDP4_LAYERMIXER_IN_CFG_PIPE5_MIXER1                     0x00800000
 #define MDP4_LAYERMIXER_IN_CFG_PIPE6__MASK                      0x07000000
 #define MDP4_LAYERMIXER_IN_CFG_PIPE6__SHIFT                     24
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE6(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE6(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE6__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE6__MASK;
 }
 #define MDP4_LAYERMIXER_IN_CFG_PIPE6_MIXER1                     0x08000000
 #define MDP4_LAYERMIXER_IN_CFG_PIPE7__MASK                      0x70000000
 #define MDP4_LAYERMIXER_IN_CFG_PIPE7__SHIFT                     28
-static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE7(enum mpd4_mixer_stage_id val)
+static inline uint32_t MDP4_LAYERMIXER_IN_CFG_PIPE7(enum mdp4_mixer_stage_id val)
 {
         return ((val) << MDP4_LAYERMIXER_IN_CFG_PIPE7__SHIFT) & MDP4_LAYERMIXER_IN_CFG_PIPE7__MASK;
 }
@@ -369,7 +369,7 @@ static inline uint32_t REG_MDP4_OVLP_STAGE(uint32_t i0, uint32_t i1) { return 0x
 static inline uint32_t REG_MDP4_OVLP_STAGE_OP(uint32_t i0, uint32_t i1) { return 0x00000000 + __offset_OVLP(i0) + __offset_STAGE(i1); }
 #define MDP4_OVLP_STAGE_OP_FG_ALPHA__MASK                       0x00000003
 #define MDP4_OVLP_STAGE_OP_FG_ALPHA__SHIFT                      0
-static inline uint32_t MDP4_OVLP_STAGE_OP_FG_ALPHA(enum mpd4_alpha_type val)
+static inline uint32_t MDP4_OVLP_STAGE_OP_FG_ALPHA(enum mdp4_alpha_type val)
 {
         return ((val) << MDP4_OVLP_STAGE_OP_FG_ALPHA__SHIFT) & MDP4_OVLP_STAGE_OP_FG_ALPHA__MASK;
 }
@@ -377,7 +377,7 @@ static inline uint32_t MDP4_OVLP_STAGE_OP_FG_ALPHA(enum mpd4_alpha_type val)
 #define MDP4_OVLP_STAGE_OP_FG_MOD_ALPHA                         0x00000008
 #define MDP4_OVLP_STAGE_OP_BG_ALPHA__MASK                       0x00000030
 #define MDP4_OVLP_STAGE_OP_BG_ALPHA__SHIFT                      4
-static inline uint32_t MDP4_OVLP_STAGE_OP_BG_ALPHA(enum mpd4_alpha_type val)
+static inline uint32_t MDP4_OVLP_STAGE_OP_BG_ALPHA(enum mdp4_alpha_type val)
 {
         return ((val) << MDP4_OVLP_STAGE_OP_BG_ALPHA__SHIFT) & MDP4_OVLP_STAGE_OP_BG_ALPHA__MASK;
 }
@@ -472,19 +472,19 @@ static inline uint32_t REG_MDP4_DMA(enum mdp4_dma i0) { return 0x00000000 + __of
 static inline uint32_t REG_MDP4_DMA_CONFIG(enum mdp4_dma i0) { return 0x00000000 + __offset_DMA(i0); }
 #define MDP4_DMA_CONFIG_G_BPC__MASK                             0x00000003
 #define MDP4_DMA_CONFIG_G_BPC__SHIFT                            0
-static inline uint32_t MDP4_DMA_CONFIG_G_BPC(enum mpd4_bpc val)
+static inline uint32_t MDP4_DMA_CONFIG_G_BPC(enum mdp4_bpc val)
 {
         return ((val) << MDP4_DMA_CONFIG_G_BPC__SHIFT) & MDP4_DMA_CONFIG_G_BPC__MASK;
 }
 #define MDP4_DMA_CONFIG_B_BPC__MASK                             0x0000000c
 #define MDP4_DMA_CONFIG_B_BPC__SHIFT                            2
-static inline uint32_t MDP4_DMA_CONFIG_B_BPC(enum mpd4_bpc val)
+static inline uint32_t MDP4_DMA_CONFIG_B_BPC(enum mdp4_bpc val)
 {
         return ((val) << MDP4_DMA_CONFIG_B_BPC__SHIFT) & MDP4_DMA_CONFIG_B_BPC__MASK;
 }
 #define MDP4_DMA_CONFIG_R_BPC__MASK                             0x00000030
 #define MDP4_DMA_CONFIG_R_BPC__SHIFT                            4
-static inline uint32_t MDP4_DMA_CONFIG_R_BPC(enum mpd4_bpc val)
+static inline uint32_t MDP4_DMA_CONFIG_R_BPC(enum mdp4_bpc val)
 {
         return ((val) << MDP4_DMA_CONFIG_R_BPC__SHIFT) & MDP4_DMA_CONFIG_R_BPC__MASK;
 }
@@ -601,9 +601,9 @@ static inline uint32_t REG_MDP4_DMA_CSC_POST_LV(enum mdp4_dma i0, uint32_t i1) {
 
 static inline uint32_t REG_MDP4_DMA_CSC_POST_LV_VAL(enum mdp4_dma i0, uint32_t i1) { return 0x00003680 + __offset_DMA(i0) + 0x4*i1; }
 
-static inline uint32_t REG_MDP4_PIPE(enum mpd4_pipe i0) { return 0x00020000 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE(enum mdp4_pipe i0) { return 0x00020000 + 0x10000*i0; }
 
-static inline uint32_t REG_MDP4_PIPE_SRC_SIZE(enum mpd4_pipe i0) { return 0x00020000 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_SRC_SIZE(enum mdp4_pipe i0) { return 0x00020000 + 0x10000*i0; }
 #define MDP4_PIPE_SRC_SIZE_HEIGHT__MASK                         0xffff0000
 #define MDP4_PIPE_SRC_SIZE_HEIGHT__SHIFT                        16
 static inline uint32_t MDP4_PIPE_SRC_SIZE_HEIGHT(uint32_t val)
@@ -617,7 +617,7 @@ static inline uint32_t MDP4_PIPE_SRC_SIZE_WIDTH(uint32_t val)
         return ((val) << MDP4_PIPE_SRC_SIZE_WIDTH__SHIFT) & MDP4_PIPE_SRC_SIZE_WIDTH__MASK;
 }
 
-static inline uint32_t REG_MDP4_PIPE_SRC_XY(enum mpd4_pipe i0) { return 0x00020004 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_SRC_XY(enum mdp4_pipe i0) { return 0x00020004 + 0x10000*i0; }
 #define MDP4_PIPE_SRC_XY_Y__MASK                                0xffff0000
 #define MDP4_PIPE_SRC_XY_Y__SHIFT                               16
 static inline uint32_t MDP4_PIPE_SRC_XY_Y(uint32_t val)
@@ -631,7 +631,7 @@ static inline uint32_t MDP4_PIPE_SRC_XY_X(uint32_t val)
         return ((val) << MDP4_PIPE_SRC_XY_X__SHIFT) & MDP4_PIPE_SRC_XY_X__MASK;
 }
 
-static inline uint32_t REG_MDP4_PIPE_DST_SIZE(enum mpd4_pipe i0) { return 0x00020008 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_DST_SIZE(enum mdp4_pipe i0) { return 0x00020008 + 0x10000*i0; }
 #define MDP4_PIPE_DST_SIZE_HEIGHT__MASK                         0xffff0000
 #define MDP4_PIPE_DST_SIZE_HEIGHT__SHIFT                        16
 static inline uint32_t MDP4_PIPE_DST_SIZE_HEIGHT(uint32_t val)
@@ -645,7 +645,7 @@ static inline uint32_t MDP4_PIPE_DST_SIZE_WIDTH(uint32_t val)
         return ((val) << MDP4_PIPE_DST_SIZE_WIDTH__SHIFT) & MDP4_PIPE_DST_SIZE_WIDTH__MASK;
 }
 
-static inline uint32_t REG_MDP4_PIPE_DST_XY(enum mpd4_pipe i0) { return 0x0002000c + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_DST_XY(enum mdp4_pipe i0) { return 0x0002000c + 0x10000*i0; }
 #define MDP4_PIPE_DST_XY_Y__MASK                                0xffff0000
 #define MDP4_PIPE_DST_XY_Y__SHIFT                               16
 static inline uint32_t MDP4_PIPE_DST_XY_Y(uint32_t val)
@@ -659,13 +659,13 @@ static inline uint32_t MDP4_PIPE_DST_XY_X(uint32_t val)
         return ((val) << MDP4_PIPE_DST_XY_X__SHIFT) & MDP4_PIPE_DST_XY_X__MASK;
 }
 
-static inline uint32_t REG_MDP4_PIPE_SRCP0_BASE(enum mpd4_pipe i0) { return 0x00020010 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_SRCP0_BASE(enum mdp4_pipe i0) { return 0x00020010 + 0x10000*i0; }
 
-static inline uint32_t REG_MDP4_PIPE_SRCP1_BASE(enum mpd4_pipe i0) { return 0x00020014 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_SRCP1_BASE(enum mdp4_pipe i0) { return 0x00020014 + 0x10000*i0; }
 
-static inline uint32_t REG_MDP4_PIPE_SRCP2_BASE(enum mpd4_pipe i0) { return 0x00020018 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_SRCP2_BASE(enum mdp4_pipe i0) { return 0x00020018 + 0x10000*i0; }
 
-static inline uint32_t REG_MDP4_PIPE_SRC_STRIDE_A(enum mpd4_pipe i0) { return 0x00020040 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_SRC_STRIDE_A(enum mdp4_pipe i0) { return 0x00020040 + 0x10000*i0; }
 #define MDP4_PIPE_SRC_STRIDE_A_P0__MASK                         0x0000ffff
 #define MDP4_PIPE_SRC_STRIDE_A_P0__SHIFT                        0
 static inline uint32_t MDP4_PIPE_SRC_STRIDE_A_P0(uint32_t val)
@@ -679,7 +679,7 @@ static inline uint32_t MDP4_PIPE_SRC_STRIDE_A_P1(uint32_t val)
         return ((val) << MDP4_PIPE_SRC_STRIDE_A_P1__SHIFT) & MDP4_PIPE_SRC_STRIDE_A_P1__MASK;
 }
 
-static inline uint32_t REG_MDP4_PIPE_SRC_STRIDE_B(enum mpd4_pipe i0) { return 0x00020044 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_SRC_STRIDE_B(enum mdp4_pipe i0) { return 0x00020044 + 0x10000*i0; }
 #define MDP4_PIPE_SRC_STRIDE_B_P2__MASK                         0x0000ffff
 #define MDP4_PIPE_SRC_STRIDE_B_P2__SHIFT                        0
 static inline uint32_t MDP4_PIPE_SRC_STRIDE_B_P2(uint32_t val)
@@ -693,7 +693,7 @@ static inline uint32_t MDP4_PIPE_SRC_STRIDE_B_P3(uint32_t val)
         return ((val) << MDP4_PIPE_SRC_STRIDE_B_P3__SHIFT) & MDP4_PIPE_SRC_STRIDE_B_P3__MASK;
 }
 
-static inline uint32_t REG_MDP4_PIPE_FRAME_SIZE(enum mpd4_pipe i0) { return 0x00020048 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_FRAME_SIZE(enum mdp4_pipe i0) { return 0x00020048 + 0x10000*i0; }
 #define MDP4_PIPE_FRAME_SIZE_HEIGHT__MASK                       0xffff0000
 #define MDP4_PIPE_FRAME_SIZE_HEIGHT__SHIFT                      16
 static inline uint32_t MDP4_PIPE_FRAME_SIZE_HEIGHT(uint32_t val)
@@ -707,28 +707,28 @@ static inline uint32_t MDP4_PIPE_FRAME_SIZE_WIDTH(uint32_t val)
         return ((val) << MDP4_PIPE_FRAME_SIZE_WIDTH__SHIFT) & MDP4_PIPE_FRAME_SIZE_WIDTH__MASK;
 }
 
-static inline uint32_t REG_MDP4_PIPE_SRC_FORMAT(enum mpd4_pipe i0) { return 0x00020050 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_SRC_FORMAT(enum mdp4_pipe i0) { return 0x00020050 + 0x10000*i0; }
 #define MDP4_PIPE_SRC_FORMAT_G_BPC__MASK                        0x00000003
 #define MDP4_PIPE_SRC_FORMAT_G_BPC__SHIFT                       0
-static inline uint32_t MDP4_PIPE_SRC_FORMAT_G_BPC(enum mpd4_bpc val)
+static inline uint32_t MDP4_PIPE_SRC_FORMAT_G_BPC(enum mdp4_bpc val)
 {
         return ((val) << MDP4_PIPE_SRC_FORMAT_G_BPC__SHIFT) & MDP4_PIPE_SRC_FORMAT_G_BPC__MASK;
 }
 #define MDP4_PIPE_SRC_FORMAT_B_BPC__MASK                        0x0000000c
 #define MDP4_PIPE_SRC_FORMAT_B_BPC__SHIFT                       2
-static inline uint32_t MDP4_PIPE_SRC_FORMAT_B_BPC(enum mpd4_bpc val)
+static inline uint32_t MDP4_PIPE_SRC_FORMAT_B_BPC(enum mdp4_bpc val)
 {
         return ((val) << MDP4_PIPE_SRC_FORMAT_B_BPC__SHIFT) & MDP4_PIPE_SRC_FORMAT_B_BPC__MASK;
 }
 #define MDP4_PIPE_SRC_FORMAT_R_BPC__MASK                        0x00000030
 #define MDP4_PIPE_SRC_FORMAT_R_BPC__SHIFT                       4
-static inline uint32_t MDP4_PIPE_SRC_FORMAT_R_BPC(enum mpd4_bpc val)
+static inline uint32_t MDP4_PIPE_SRC_FORMAT_R_BPC(enum mdp4_bpc val)
 {
         return ((val) << MDP4_PIPE_SRC_FORMAT_R_BPC__SHIFT) & MDP4_PIPE_SRC_FORMAT_R_BPC__MASK;
 }
 #define MDP4_PIPE_SRC_FORMAT_A_BPC__MASK                        0x000000c0
 #define MDP4_PIPE_SRC_FORMAT_A_BPC__SHIFT                       6
-static inline uint32_t MDP4_PIPE_SRC_FORMAT_A_BPC(enum mpd4_bpc_alpha val)
+static inline uint32_t MDP4_PIPE_SRC_FORMAT_A_BPC(enum mdp4_bpc_alpha val)
 {
         return ((val) << MDP4_PIPE_SRC_FORMAT_A_BPC__SHIFT) & MDP4_PIPE_SRC_FORMAT_A_BPC__MASK;
 }
@@ -750,7 +750,7 @@ static inline uint32_t MDP4_PIPE_SRC_FORMAT_UNPACK_COUNT(uint32_t val)
 #define MDP4_PIPE_SRC_FORMAT_UNPACK_ALIGN_MSB                   0x00040000
 #define MDP4_PIPE_SRC_FORMAT_SOLID_FILL                         0x00400000
 
-static inline uint32_t REG_MDP4_PIPE_SRC_UNPACK(enum mpd4_pipe i0) { return 0x00020054 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_SRC_UNPACK(enum mdp4_pipe i0) { return 0x00020054 + 0x10000*i0; }
 #define MDP4_PIPE_SRC_UNPACK_ELEM0__MASK                        0x000000ff
 #define MDP4_PIPE_SRC_UNPACK_ELEM0__SHIFT                       0
 static inline uint32_t MDP4_PIPE_SRC_UNPACK_ELEM0(uint32_t val)
@@ -776,7 +776,7 @@ static inline uint32_t MDP4_PIPE_SRC_UNPACK_ELEM3(uint32_t val)
         return ((val) << MDP4_PIPE_SRC_UNPACK_ELEM3__SHIFT) & MDP4_PIPE_SRC_UNPACK_ELEM3__MASK;
 }
 
-static inline uint32_t REG_MDP4_PIPE_OP_MODE(enum mpd4_pipe i0) { return 0x00020058 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_OP_MODE(enum mdp4_pipe i0) { return 0x00020058 + 0x10000*i0; }
 #define MDP4_PIPE_OP_MODE_SCALEX_EN                             0x00000001
 #define MDP4_PIPE_OP_MODE_SCALEY_EN                             0x00000002
 #define MDP4_PIPE_OP_MODE_SRC_YCBCR                             0x00000200
@@ -789,36 +789,36 @@ static inline uint32_t REG_MDP4_PIPE_OP_MODE(enum mpd4_pipe i0) { return 0x00020
 #define MDP4_PIPE_OP_MODE_DEINT_EN                              0x00040000
 #define MDP4_PIPE_OP_MODE_DEINT_ODD_REF                         0x00080000
 
-static inline uint32_t REG_MDP4_PIPE_PHASEX_STEP(enum mpd4_pipe i0) { return 0x0002005c + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_PHASEX_STEP(enum mdp4_pipe i0) { return 0x0002005c + 0x10000*i0; }
 
-static inline uint32_t REG_MDP4_PIPE_PHASEY_STEP(enum mpd4_pipe i0) { return 0x00020060 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_PHASEY_STEP(enum mdp4_pipe i0) { return 0x00020060 + 0x10000*i0; }
 
-static inline uint32_t REG_MDP4_PIPE_FETCH_CONFIG(enum mpd4_pipe i0) { return 0x00021004 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_FETCH_CONFIG(enum mdp4_pipe i0) { return 0x00021004 + 0x10000*i0; }
 
-static inline uint32_t REG_MDP4_PIPE_SOLID_COLOR(enum mpd4_pipe i0) { return 0x00021008 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_SOLID_COLOR(enum mdp4_pipe i0) { return 0x00021008 + 0x10000*i0; }
 
-static inline uint32_t REG_MDP4_PIPE_CSC(enum mpd4_pipe i0) { return 0x00024000 + 0x10000*i0; }
+static inline uint32_t REG_MDP4_PIPE_CSC(enum mdp4_pipe i0) { return 0x00024000 + 0x10000*i0; }
 
 
-static inline uint32_t REG_MDP4_PIPE_CSC_MV(enum mpd4_pipe i0, uint32_t i1) { return 0x00024400 + 0x10000*i0 + 0x4*i1; }
+static inline uint32_t REG_MDP4_PIPE_CSC_MV(enum mdp4_pipe i0, uint32_t i1) { return 0x00024400 + 0x10000*i0 + 0x4*i1; }
 
-static inline uint32_t REG_MDP4_PIPE_CSC_MV_VAL(enum mpd4_pipe i0, uint32_t i1) { return 0x00024400 + 0x10000*i0 + 0x4*i1; }
+static inline uint32_t REG_MDP4_PIPE_CSC_MV_VAL(enum mdp4_pipe i0, uint32_t i1) { return 0x00024400 + 0x10000*i0 + 0x4*i1; }
 
-static inline uint32_t REG_MDP4_PIPE_CSC_PRE_BV(enum mpd4_pipe i0, uint32_t i1) { return 0x00024500 + 0x10000*i0 + 0x4*i1; }
+static inline uint32_t REG_MDP4_PIPE_CSC_PRE_BV(enum mdp4_pipe i0, uint32_t i1) { return 0x00024500 + 0x10000*i0 + 0x4*i1; }
 
-static inline uint32_t REG_MDP4_PIPE_CSC_PRE_BV_VAL(enum mpd4_pipe i0, uint32_t i1) { return 0x00024500 + 0x10000*i0 + 0x4*i1; }
+static inline uint32_t REG_MDP4_PIPE_CSC_PRE_BV_VAL(enum mdp4_pipe i0, uint32_t i1) { return 0x00024500 + 0x10000*i0 + 0x4*i1; }
 
-static inline uint32_t REG_MDP4_PIPE_CSC_POST_BV(enum mpd4_pipe i0, uint32_t i1) { return 0x00024580 + 0x10000*i0 + 0x4*i1; }
+static inline uint32_t REG_MDP4_PIPE_CSC_POST_BV(enum mdp4_pipe i0, uint32_t i1) { return 0x00024580 + 0x10000*i0 + 0x4*i1; }
 
-static inline uint32_t REG_MDP4_PIPE_CSC_POST_BV_VAL(enum mpd4_pipe i0, uint32_t i1) { return 0x00024580 + 0x10000*i0 + 0x4*i1; }
+static inline uint32_t REG_MDP4_PIPE_CSC_POST_BV_VAL(enum mdp4_pipe i0, uint32_t i1) { return 0x00024580 + 0x10000*i0 + 0x4*i1; }
 
-static inline uint32_t REG_MDP4_PIPE_CSC_PRE_LV(enum mpd4_pipe i0, uint32_t i1) { return 0x00024600 + 0x10000*i0 + 0x4*i1; }
+static inline uint32_t REG_MDP4_PIPE_CSC_PRE_LV(enum mdp4_pipe i0, uint32_t i1) { return 0x00024600 + 0x10000*i0 + 0x4*i1; }
 
-static inline uint32_t REG_MDP4_PIPE_CSC_PRE_LV_VAL(enum mpd4_pipe i0, uint32_t i1) { return 0x00024600 + 0x10000*i0 + 0x4*i1; }
+static inline uint32_t REG_MDP4_PIPE_CSC_PRE_LV_VAL(enum mdp4_pipe i0, uint32_t i1) { return 0x00024600 + 0x10000*i0 + 0x4*i1; }
 
-static inline uint32_t REG_MDP4_PIPE_CSC_POST_LV(enum mpd4_pipe i0, uint32_t i1) { return 0x00024680 + 0x10000*i0 + 0x4*i1; }
+static inline uint32_t REG_MDP4_PIPE_CSC_POST_LV(enum mdp4_pipe i0, uint32_t i1) { return 0x00024680 + 0x10000*i0 + 0x4*i1; }
 
-static inline uint32_t REG_MDP4_PIPE_CSC_POST_LV_VAL(enum mpd4_pipe i0, uint32_t i1) { return 0x00024680 + 0x10000*i0 + 0x4*i1; }
+static inline uint32_t REG_MDP4_PIPE_CSC_POST_LV_VAL(enum mdp4_pipe i0, uint32_t i1) { return 0x00024680 + 0x10000*i0 + 0x4*i1; }
 
 #define REG_MDP4_LCDC                                           0x000c0000
 
diff --git a/drivers/gpu/drm/msm/mdp4/mdp4_crtc.c b/drivers/gpu/drm/msm/mdp4/mdp4_crtc.c
index de6bea297cda..019d530187ff 100644
--- a/drivers/gpu/drm/msm/mdp4/mdp4_crtc.c
+++ b/drivers/gpu/drm/msm/mdp4/mdp4_crtc.c
@@ -26,6 +26,7 @@ struct mdp4_crtc {
         struct drm_crtc base;
         char name[8];
         struct drm_plane *plane;
+        struct drm_plane *planes[8];
         int id;
         int ovlp;
         enum mdp4_dma dma;
@@ -50,7 +51,11 @@ struct mdp4_crtc {
 
         /* if there is a pending flip, these will be non-null: */
         struct drm_pending_vblank_event *event;
-        struct work_struct pageflip_work;
+        struct msm_fence_cb pageflip_cb;
+
+#define PENDING_CURSOR 0x1
+#define PENDING_FLIP   0x2
+        atomic_t pending;
 
         /* the fb that we currently hold a scanout ref to: */
         struct drm_framebuffer *fb;
@@ -92,7 +97,8 @@ static void update_fb(struct drm_crtc *crtc, bool async,
         }
 }
 
-static void complete_flip(struct drm_crtc *crtc, bool canceled)
+/* if file!=NULL, this is preclose potential cancel-flip path */
+static void complete_flip(struct drm_crtc *crtc, struct drm_file *file)
 {
         struct mdp4_crtc *mdp4_crtc = to_mdp4_crtc(crtc);
         struct drm_device *dev = crtc->dev;
@@ -102,11 +108,14 @@ static void complete_flip(struct drm_crtc *crtc, bool canceled)
         spin_lock_irqsave(&dev->event_lock, flags);
         event = mdp4_crtc->event;
         if (event) {
-                mdp4_crtc->event = NULL;
-                if (canceled)
-                        event->base.destroy(&event->base);
-                else
+                /* if regular vblank case (!file) or if cancel-flip from
+                 * preclose on file that requested flip, then send the
+                 * event:
+                 */
+                if (!file || (event->base.file_priv == file)) {
+                        mdp4_crtc->event = NULL;
                         drm_send_vblank_event(dev, mdp4_crtc->id, event);
+                }
         }
         spin_unlock_irqrestore(&dev->event_lock, flags);
 }
@@ -115,9 +124,15 @@ static void crtc_flush(struct drm_crtc *crtc)
 {
         struct mdp4_crtc *mdp4_crtc = to_mdp4_crtc(crtc);
         struct mdp4_kms *mdp4_kms = get_kms(crtc);
-        uint32_t flush = 0;
+        uint32_t i, flush = 0;
 
-        flush |= pipe2flush(mdp4_plane_pipe(mdp4_crtc->plane));
+        for (i = 0; i < ARRAY_SIZE(mdp4_crtc->planes); i++) {
+                struct drm_plane *plane = mdp4_crtc->planes[i];
+                if (plane) {
+                        enum mdp4_pipe pipe_id = mdp4_plane_pipe(plane);
+                        flush |= pipe2flush(pipe_id);
+                }
+        }
         flush |= ovlp2flush(mdp4_crtc->ovlp);
 
         DBG("%s: flush=%08x", mdp4_crtc->name, flush);
@@ -125,17 +140,29 @@ static void crtc_flush(struct drm_crtc *crtc)
         mdp4_write(mdp4_kms, REG_MDP4_OVERLAY_FLUSH, flush);
 }
 
-static void pageflip_worker(struct work_struct *work)
+static void request_pending(struct drm_crtc *crtc, uint32_t pending)
+{
+        struct mdp4_crtc *mdp4_crtc = to_mdp4_crtc(crtc);
+
+        atomic_or(pending, &mdp4_crtc->pending);
+        mdp4_irq_register(get_kms(crtc), &mdp4_crtc->vblank);
+}
+
+static void pageflip_cb(struct msm_fence_cb *cb)
 {
         struct mdp4_crtc *mdp4_crtc =
-                container_of(work, struct mdp4_crtc, pageflip_work);
+                container_of(cb, struct mdp4_crtc, pageflip_cb);
         struct drm_crtc *crtc = &mdp4_crtc->base;
+        struct drm_framebuffer *fb = crtc->fb;
 
-        mdp4_plane_set_scanout(mdp4_crtc->plane, crtc->fb);
+        if (!fb)
+                return;
+
+        mdp4_plane_set_scanout(mdp4_crtc->plane, fb);
         crtc_flush(crtc);
 
         /* enable vblank to complete flip: */
-        mdp4_irq_register(get_kms(crtc), &mdp4_crtc->vblank);
+        request_pending(crtc, PENDING_FLIP);
 }
 
 static void unref_fb_worker(struct drm_flip_work *work, void *val)
@@ -205,67 +232,69 @@ static void blend_setup(struct drm_crtc *crtc)
         struct mdp4_kms *mdp4_kms = get_kms(crtc);
         int i, ovlp = mdp4_crtc->ovlp;
         uint32_t mixer_cfg = 0;
-
-        /*
-         * This probably would also need to be triggered by any attached
-         * plane when it changes.. for now since we are only using a single
-         * private plane, the configuration is hard-coded:
-         */
+        static const enum mdp4_mixer_stage_id stages[] = {
+                        STAGE_BASE, STAGE0, STAGE1, STAGE2, STAGE3,
+        };
+        /* statically (for now) map planes to mixer stage (z-order): */
+        static const int idxs[] = {
+                        [VG1]  = 1,
+                        [VG2]  = 2,
+                        [RGB1] = 0,
+                        [RGB2] = 0,
+                        [RGB3] = 0,
+                        [VG3]  = 3,
+                        [VG4]  = 4,
+
+        };
+        bool alpha[4]= { false, false, false, false };
 
         mdp4_write(mdp4_kms, REG_MDP4_OVLP_TRANSP_LOW0(ovlp), 0);
         mdp4_write(mdp4_kms, REG_MDP4_OVLP_TRANSP_LOW1(ovlp), 0);
         mdp4_write(mdp4_kms, REG_MDP4_OVLP_TRANSP_HIGH0(ovlp), 0);
         mdp4_write(mdp4_kms, REG_MDP4_OVLP_TRANSP_HIGH1(ovlp), 0);
 
+        /* TODO single register for all CRTCs, so this won't work properly
+         * when multiple CRTCs are active..
+         */
+        for (i = 0; i < ARRAY_SIZE(mdp4_crtc->planes); i++) {
+                struct drm_plane *plane = mdp4_crtc->planes[i];
+                if (plane) {
+                        enum mdp4_pipe pipe_id = mdp4_plane_pipe(plane);
+                        int idx = idxs[pipe_id];
+                        if (idx > 0) {
+                                const struct mdp4_format *format =
+                                        to_mdp4_format(msm_framebuffer_format(plane->fb));
+                                alpha[idx-1] = format->alpha_enable;
+                        }
+                        mixer_cfg |= mixercfg(mdp4_crtc->mixer, pipe_id, stages[idx]);
+                }
+        }
+
+        /* this shouldn't happen.. and seems to cause underflow: */
+        WARN_ON(!mixer_cfg);
+
         for (i = 0; i < 4; i++) {
-                mdp4_write(mdp4_kms, REG_MDP4_OVLP_STAGE_FG_ALPHA(ovlp, i), 0);
-                mdp4_write(mdp4_kms, REG_MDP4_OVLP_STAGE_BG_ALPHA(ovlp, i), 0);
-                mdp4_write(mdp4_kms, REG_MDP4_OVLP_STAGE_OP(ovlp, i),
-                                MDP4_OVLP_STAGE_OP_FG_ALPHA(FG_CONST) |
-                                MDP4_OVLP_STAGE_OP_BG_ALPHA(BG_CONST));
-                mdp4_write(mdp4_kms, REG_MDP4_OVLP_STAGE_CO3(ovlp, i), 0);
+                uint32_t op;
+
+                if (alpha[i]) {
+                        op = MDP4_OVLP_STAGE_OP_FG_ALPHA(FG_PIXEL) |
+                                        MDP4_OVLP_STAGE_OP_BG_ALPHA(FG_PIXEL) |
+                                        MDP4_OVLP_STAGE_OP_BG_INV_ALPHA;
+                } else {
+                        op = MDP4_OVLP_STAGE_OP_FG_ALPHA(FG_CONST) |
+                                        MDP4_OVLP_STAGE_OP_BG_ALPHA(BG_CONST);
+                }
+
+                mdp4_write(mdp4_kms, REG_MDP4_OVLP_STAGE_FG_ALPHA(ovlp, i), 0xff);
+                mdp4_write(mdp4_kms, REG_MDP4_OVLP_STAGE_BG_ALPHA(ovlp, i), 0x00);
+                mdp4_write(mdp4_kms, REG_MDP4_OVLP_STAGE_OP(ovlp, i), op);
+                mdp4_write(mdp4_kms, REG_MDP4_OVLP_STAGE_CO3(ovlp, i), 1);
                 mdp4_write(mdp4_kms, REG_MDP4_OVLP_STAGE_TRANSP_LOW0(ovlp, i), 0);
                 mdp4_write(mdp4_kms, REG_MDP4_OVLP_STAGE_TRANSP_LOW1(ovlp, i), 0);
                 mdp4_write(mdp4_kms, REG_MDP4_OVLP_STAGE_TRANSP_HIGH0(ovlp, i), 0);
                 mdp4_write(mdp4_kms, REG_MDP4_OVLP_STAGE_TRANSP_HIGH1(ovlp, i), 0);
         }
 
-        /* TODO single register for all CRTCs, so this won't work properly
-         * when multiple CRTCs are active..
-         */
-        switch (mdp4_plane_pipe(mdp4_crtc->plane)) {
-        case VG1:
-                mixer_cfg = MDP4_LAYERMIXER_IN_CFG_PIPE0(STAGE_BASE) |
-                        COND(mdp4_crtc->mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE0_MIXER1);
-                break;
-        case VG2:
-                mixer_cfg = MDP4_LAYERMIXER_IN_CFG_PIPE1(STAGE_BASE) |
-                        COND(mdp4_crtc->mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE1_MIXER1);
-                break;
-        case RGB1:
-                mixer_cfg = MDP4_LAYERMIXER_IN_CFG_PIPE2(STAGE_BASE) |
-                        COND(mdp4_crtc->mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE2_MIXER1);
-                break;
-        case RGB2:
-                mixer_cfg = MDP4_LAYERMIXER_IN_CFG_PIPE3(STAGE_BASE) |
-                        COND(mdp4_crtc->mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE3_MIXER1);
-                break;
-        case RGB3:
-                mixer_cfg = MDP4_LAYERMIXER_IN_CFG_PIPE4(STAGE_BASE) |
-                        COND(mdp4_crtc->mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE4_MIXER1);
-                break;
-        case VG3:
-                mixer_cfg = MDP4_LAYERMIXER_IN_CFG_PIPE5(STAGE_BASE) |
-                        COND(mdp4_crtc->mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE5_MIXER1);
-                break;
-        case VG4:
-                mixer_cfg = MDP4_LAYERMIXER_IN_CFG_PIPE6(STAGE_BASE) |
-                        COND(mdp4_crtc->mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE6_MIXER1);
-                break;
-        default:
-                WARN_ON("invalid pipe");
-                break;
-        }
         mdp4_write(mdp4_kms, REG_MDP4_LAYERMIXER_IN_CFG, mixer_cfg);
 }
 
@@ -377,6 +406,7 @@ static int mdp4_crtc_page_flip(struct drm_crtc *crtc,
         struct mdp4_crtc *mdp4_crtc = to_mdp4_crtc(crtc);
         struct drm_device *dev = crtc->dev;
         struct drm_gem_object *obj;
+        unsigned long flags;
 
         if (mdp4_crtc->event) {
                 dev_err(dev->dev, "already pending flip!\n");
@@ -385,11 +415,13 @@ static int mdp4_crtc_page_flip(struct drm_crtc *crtc,
 
         obj = msm_framebuffer_bo(new_fb, 0);
 
+        spin_lock_irqsave(&dev->event_lock, flags);
         mdp4_crtc->event = event;
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+
         update_fb(crtc, true, new_fb);
 
-        return msm_gem_queue_inactive_work(obj,
-                        &mdp4_crtc->pageflip_work);
+        return msm_gem_queue_inactive_cb(obj, &mdp4_crtc->pageflip_cb);
 }
 
 static int mdp4_crtc_set_property(struct drm_crtc *crtc,
@@ -498,6 +530,8 @@ static int mdp4_crtc_cursor_set(struct drm_crtc *crtc,
                 drm_gem_object_unreference_unlocked(old_bo);
         }
 
+        request_pending(crtc, PENDING_CURSOR);
+
         return 0;
 
 fail:
@@ -542,13 +576,21 @@ static void mdp4_crtc_vblank_irq(struct mdp4_irq *irq, uint32_t irqstatus)
         struct mdp4_crtc *mdp4_crtc = container_of(irq, struct mdp4_crtc, vblank);
         struct drm_crtc *crtc = &mdp4_crtc->base;
         struct msm_drm_private *priv = crtc->dev->dev_private;
+        unsigned pending;
 
-        update_cursor(crtc);
-        complete_flip(crtc, false);
         mdp4_irq_unregister(get_kms(crtc), &mdp4_crtc->vblank);
 
-        drm_flip_work_commit(&mdp4_crtc->unref_fb_work, priv->wq);
-        drm_flip_work_commit(&mdp4_crtc->unref_cursor_work, priv->wq);
+        pending = atomic_xchg(&mdp4_crtc->pending, 0);
+
+        if (pending & PENDING_FLIP) {
+                complete_flip(crtc, NULL);
+                drm_flip_work_commit(&mdp4_crtc->unref_fb_work, priv->wq);
+        }
+
+        if (pending & PENDING_CURSOR) {
+                update_cursor(crtc);
+                drm_flip_work_commit(&mdp4_crtc->unref_cursor_work, priv->wq);
+        }
 }
 
 static void mdp4_crtc_err_irq(struct mdp4_irq *irq, uint32_t irqstatus)
@@ -565,9 +607,10 @@ uint32_t mdp4_crtc_vblank(struct drm_crtc *crtc)
         return mdp4_crtc->vblank.irqmask;
 }
 
-void mdp4_crtc_cancel_pending_flip(struct drm_crtc *crtc)
+void mdp4_crtc_cancel_pending_flip(struct drm_crtc *crtc, struct drm_file *file)
 {
-        complete_flip(crtc, true);
+        DBG("cancel: %p", file);
+        complete_flip(crtc, file);
 }
 
 /* set dma config, ie. the format the encoder wants. */
@@ -622,6 +665,32 @@ void mdp4_crtc_set_intf(struct drm_crtc *crtc, enum mdp4_intf intf)
         mdp4_write(mdp4_kms, REG_MDP4_DISP_INTF_SEL, intf_sel);
 }
 
+static void set_attach(struct drm_crtc *crtc, enum mdp4_pipe pipe_id,
+                struct drm_plane *plane)
+{
+        struct mdp4_crtc *mdp4_crtc = to_mdp4_crtc(crtc);
+
+        BUG_ON(pipe_id >= ARRAY_SIZE(mdp4_crtc->planes));
+
+        if (mdp4_crtc->planes[pipe_id] == plane)
+                return;
+
+        mdp4_crtc->planes[pipe_id] = plane;
+        blend_setup(crtc);
+        if (mdp4_crtc->enabled && (plane != mdp4_crtc->plane))
+                crtc_flush(crtc);
+}
+
+void mdp4_crtc_attach(struct drm_crtc *crtc, struct drm_plane *plane)
+{
+        set_attach(crtc, mdp4_plane_pipe(plane), plane);
+}
+
+void mdp4_crtc_detach(struct drm_crtc *crtc, struct drm_plane *plane)
+{
+        set_attach(crtc, mdp4_plane_pipe(plane), NULL);
+}
+
 static const char *dma_names[] = {
                 "DMA_P", "DMA_S", "DMA_E",
 };
@@ -644,7 +713,6 @@ struct drm_crtc *mdp4_crtc_init(struct drm_device *dev,
         crtc = &mdp4_crtc->base;
 
         mdp4_crtc->plane = plane;
-        mdp4_crtc->plane->crtc = crtc;
 
         mdp4_crtc->ovlp = ovlp_id;
         mdp4_crtc->dma = dma_id;
@@ -668,7 +736,7 @@ struct drm_crtc *mdp4_crtc_init(struct drm_device *dev,
         ret = drm_flip_work_init(&mdp4_crtc->unref_cursor_work, 64,
                         "unref cursor", unref_cursor_worker);
 
-        INIT_WORK(&mdp4_crtc->pageflip_work, pageflip_worker);
+        INIT_FENCE_CB(&mdp4_crtc->pageflip_cb, pageflip_cb);
 
         drm_crtc_init(dev, crtc, &mdp4_crtc_funcs);
         drm_crtc_helper_add(crtc, &mdp4_crtc_helper_funcs);
diff --git a/drivers/gpu/drm/msm/mdp4/mdp4_format.c b/drivers/gpu/drm/msm/mdp4/mdp4_format.c
index 7b645f2e837a..17330b0927b2 100644
--- a/drivers/gpu/drm/msm/mdp4/mdp4_format.c
+++ b/drivers/gpu/drm/msm/mdp4/mdp4_format.c
@@ -44,6 +44,22 @@ static const struct mdp4_format formats[] = {
         FMT(BGR565,   0, 5, 6, 5,  2, 0, 1, 0,  false,  true,  2,  3),
 };
 
+uint32_t mdp4_get_formats(enum mdp4_pipe pipe_id, uint32_t *pixel_formats,
+                uint32_t max_formats)
+{
+        uint32_t i;
+        for (i = 0; i < ARRAY_SIZE(formats); i++) {
+                const struct mdp4_format *f = &formats[i];
+
+                if (i == max_formats)
+                        break;
+
+                pixel_formats[i] = f->base.pixel_format;
+        }
+
+        return i;
+}
+
 const struct msm_format *mdp4_get_format(struct msm_kms *kms, uint32_t format)
 {
         int i;
diff --git a/drivers/gpu/drm/msm/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/mdp4/mdp4_kms.c
index bc7fd11ad8be..8972ac35a43d 100644
--- a/drivers/gpu/drm/msm/mdp4/mdp4_kms.c
+++ b/drivers/gpu/drm/msm/mdp4/mdp4_kms.c
@@ -135,7 +135,7 @@ static void mdp4_preclose(struct msm_kms *kms, struct drm_file *file)
         unsigned i;
 
         for (i = 0; i < priv->num_crtcs; i++)
-                mdp4_crtc_cancel_pending_flip(priv->crtcs[i]);
+                mdp4_crtc_cancel_pending_flip(priv->crtcs[i], file);
 }
 
 static void mdp4_destroy(struct msm_kms *kms)
@@ -196,6 +196,23 @@ static int modeset_init(struct mdp4_kms *mdp4_kms)
          * for more than just RGB1->DMA_E->DTV->HDMI
          */
 
+        /* construct non-private planes: */
+        plane = mdp4_plane_init(dev, VG1, false);
+        if (IS_ERR(plane)) {
+                dev_err(dev->dev, "failed to construct plane for VG1\n");
+                ret = PTR_ERR(plane);
+                goto fail;
+        }
+        priv->planes[priv->num_planes++] = plane;
+
+        plane = mdp4_plane_init(dev, VG2, false);
+        if (IS_ERR(plane)) {
+                dev_err(dev->dev, "failed to construct plane for VG2\n");
+                ret = PTR_ERR(plane);
+                goto fail;
+        }
+        priv->planes[priv->num_planes++] = plane;
+
         /* the CRTCs get constructed with a private plane: */
         plane = mdp4_plane_init(dev, RGB1, true);
         if (IS_ERR(plane)) {
diff --git a/drivers/gpu/drm/msm/mdp4/mdp4_kms.h b/drivers/gpu/drm/msm/mdp4/mdp4_kms.h
index 1e83554955f3..eb015c834087 100644
--- a/drivers/gpu/drm/msm/mdp4/mdp4_kms.h
+++ b/drivers/gpu/drm/msm/mdp4/mdp4_kms.h
@@ -75,8 +75,8 @@ struct mdp4_platform_config {
 
 struct mdp4_format {
         struct msm_format base;
-        enum mpd4_bpc bpc_r, bpc_g, bpc_b;
-        enum mpd4_bpc_alpha bpc_a;
+        enum mdp4_bpc bpc_r, bpc_g, bpc_b;
+        enum mdp4_bpc_alpha bpc_a;
         uint8_t unpack[4];
         bool alpha_enable, unpack_tight;
         uint8_t cpp, unpack_count;
@@ -93,7 +93,7 @@ static inline u32 mdp4_read(struct mdp4_kms *mdp4_kms, u32 reg)
         return msm_readl(mdp4_kms->mmio + reg);
 }
 
-static inline uint32_t pipe2flush(enum mpd4_pipe pipe)
+static inline uint32_t pipe2flush(enum mdp4_pipe pipe)
 {
         switch (pipe) {
         case VG1:      return MDP4_OVERLAY_FLUSH_VG1;
@@ -133,6 +133,48 @@ static inline uint32_t dma2err(enum mdp4_dma dma)
         }
 }
 
+static inline uint32_t mixercfg(int mixer, enum mdp4_pipe pipe,
+                enum mdp4_mixer_stage_id stage)
+{
+        uint32_t mixer_cfg = 0;
+
+        switch (pipe) {
+        case VG1:
+                mixer_cfg = MDP4_LAYERMIXER_IN_CFG_PIPE0(stage) |
+                        COND(mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE0_MIXER1);
+                break;
+        case VG2:
+                mixer_cfg = MDP4_LAYERMIXER_IN_CFG_PIPE1(stage) |
+                        COND(mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE1_MIXER1);
+                break;
+        case RGB1:
+                mixer_cfg = MDP4_LAYERMIXER_IN_CFG_PIPE2(stage) |
+                        COND(mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE2_MIXER1);
+                break;
+        case RGB2:
+                mixer_cfg = MDP4_LAYERMIXER_IN_CFG_PIPE3(stage) |
+                        COND(mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE3_MIXER1);
+                break;
+        case RGB3:
+                mixer_cfg = MDP4_LAYERMIXER_IN_CFG_PIPE4(stage) |
+                        COND(mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE4_MIXER1);
+                break;
+        case VG3:
+                mixer_cfg = MDP4_LAYERMIXER_IN_CFG_PIPE5(stage) |
+                        COND(mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE5_MIXER1);
+                break;
+        case VG4:
+                mixer_cfg = MDP4_LAYERMIXER_IN_CFG_PIPE6(stage) |
+                        COND(mixer == 1, MDP4_LAYERMIXER_IN_CFG_PIPE6_MIXER1);
+                break;
+        default:
+                WARN_ON("invalid pipe");
+                break;
+        }
+
+        return mixer_cfg;
+}
+
 int mdp4_disable(struct mdp4_kms *mdp4_kms);
 int mdp4_enable(struct mdp4_kms *mdp4_kms);
 
@@ -146,6 +188,8 @@ void mdp4_irq_unregister(struct mdp4_kms *mdp4_kms, struct mdp4_irq *irq);
 int mdp4_enable_vblank(struct msm_kms *kms, struct drm_crtc *crtc);
 void mdp4_disable_vblank(struct msm_kms *kms, struct drm_crtc *crtc);
 
+uint32_t mdp4_get_formats(enum mdp4_pipe pipe_id, uint32_t *formats,
+                uint32_t max_formats);
 const struct msm_format *mdp4_get_format(struct msm_kms *kms, uint32_t format);
 
 void mdp4_plane_install_properties(struct drm_plane *plane,
@@ -158,14 +202,16 @@ int mdp4_plane_mode_set(struct drm_plane *plane,
                 unsigned int crtc_w, unsigned int crtc_h,
                 uint32_t src_x, uint32_t src_y,
                 uint32_t src_w, uint32_t src_h);
-enum mpd4_pipe mdp4_plane_pipe(struct drm_plane *plane);
+enum mdp4_pipe mdp4_plane_pipe(struct drm_plane *plane);
 struct drm_plane *mdp4_plane_init(struct drm_device *dev,
-                enum mpd4_pipe pipe_id, bool private_plane);
+                enum mdp4_pipe pipe_id, bool private_plane);
 
 uint32_t mdp4_crtc_vblank(struct drm_crtc *crtc);
-void mdp4_crtc_cancel_pending_flip(struct drm_crtc *crtc);
+void mdp4_crtc_cancel_pending_flip(struct drm_crtc *crtc, struct drm_file *file);
 void mdp4_crtc_set_config(struct drm_crtc *crtc, uint32_t config);
 void mdp4_crtc_set_intf(struct drm_crtc *crtc, enum mdp4_intf intf);
+void mdp4_crtc_attach(struct drm_crtc *crtc, struct drm_plane *plane);
+void mdp4_crtc_detach(struct drm_crtc *crtc, struct drm_plane *plane);
 struct drm_crtc *mdp4_crtc_init(struct drm_device *dev,
                 struct drm_plane *plane, int id, int ovlp_id,
                 enum mdp4_dma dma_id);
diff --git a/drivers/gpu/drm/msm/mdp4/mdp4_plane.c b/drivers/gpu/drm/msm/mdp4/mdp4_plane.c
index 3468229d58b3..0f0af243f6fc 100644
--- a/drivers/gpu/drm/msm/mdp4/mdp4_plane.c
+++ b/drivers/gpu/drm/msm/mdp4/mdp4_plane.c
@@ -22,7 +22,7 @@ struct mdp4_plane {
         struct drm_plane base;
         const char *name;
 
-        enum mpd4_pipe pipe;
+        enum mdp4_pipe pipe;
 
         uint32_t nformats;
         uint32_t formats[32];
@@ -61,7 +61,9 @@ static int mdp4_plane_update(struct drm_plane *plane,
 static int mdp4_plane_disable(struct drm_plane *plane)
 {
         struct mdp4_plane *mdp4_plane = to_mdp4_plane(plane);
-        DBG("%s: TODO", mdp4_plane->name); // XXX
+        DBG("%s: disable", mdp4_plane->name);
+        if (plane->crtc)
+                mdp4_crtc_detach(plane->crtc, plane);
         return 0;
 }
 
@@ -101,7 +103,7 @@ void mdp4_plane_set_scanout(struct drm_plane *plane,
 {
         struct mdp4_plane *mdp4_plane = to_mdp4_plane(plane);
         struct mdp4_kms *mdp4_kms = get_kms(plane);
-        enum mpd4_pipe pipe = mdp4_plane->pipe;
+        enum mdp4_pipe pipe = mdp4_plane->pipe;
         uint32_t iova;
 
         mdp4_write(mdp4_kms, REG_MDP4_PIPE_SRC_STRIDE_A(pipe),
@@ -129,7 +131,7 @@ int mdp4_plane_mode_set(struct drm_plane *plane,
 {
         struct mdp4_plane *mdp4_plane = to_mdp4_plane(plane);
         struct mdp4_kms *mdp4_kms = get_kms(plane);
-        enum mpd4_pipe pipe = mdp4_plane->pipe;
+        enum mdp4_pipe pipe = mdp4_plane->pipe;
         const struct mdp4_format *format;
         uint32_t op_mode = 0;
         uint32_t phasex_step = MDP4_VG_PHASE_STEP_DEFAULT;
@@ -141,6 +143,10 @@ int mdp4_plane_mode_set(struct drm_plane *plane,
         src_w = src_w >> 16;
         src_h = src_h >> 16;
 
+        DBG("%s: FB[%u] %u,%u,%u,%u -> CRTC[%u] %d,%d,%u,%u", mdp4_plane->name,
+                        fb->base.id, src_x, src_y, src_w, src_h,
+                        crtc->base.id, crtc_x, crtc_y, crtc_w, crtc_h);
+
         if (src_w != crtc_w) {
                 op_mode |= MDP4_PIPE_OP_MODE_SCALEX_EN;
                 /* TODO calc phasex_step */
@@ -191,7 +197,8 @@ int mdp4_plane_mode_set(struct drm_plane *plane,
         mdp4_write(mdp4_kms, REG_MDP4_PIPE_PHASEX_STEP(pipe), phasex_step);
         mdp4_write(mdp4_kms, REG_MDP4_PIPE_PHASEY_STEP(pipe), phasey_step);
 
-        plane->crtc = crtc;
+        /* TODO detach from old crtc (if we had more than one) */
+        mdp4_crtc_attach(crtc, plane);
 
         return 0;
 }
@@ -202,7 +209,7 @@ static const char *pipe_names[] = {
                 "VG3", "VG4",
 };
 
-enum mpd4_pipe mdp4_plane_pipe(struct drm_plane *plane)
+enum mdp4_pipe mdp4_plane_pipe(struct drm_plane *plane)
 {
         struct mdp4_plane *mdp4_plane = to_mdp4_plane(plane);
         return mdp4_plane->pipe;
@@ -210,9 +217,8 @@ enum mpd4_pipe mdp4_plane_pipe(struct drm_plane *plane)
 
 /* initialize plane */
 struct drm_plane *mdp4_plane_init(struct drm_device *dev,
-                enum mpd4_pipe pipe_id, bool private_plane)
+                enum mdp4_pipe pipe_id, bool private_plane)
 {
-        struct msm_drm_private *priv = dev->dev_private;
         struct drm_plane *plane = NULL;
         struct mdp4_plane *mdp4_plane;
         int ret;
@@ -228,8 +234,12 @@ struct drm_plane *mdp4_plane_init(struct drm_device *dev,
         mdp4_plane->pipe = pipe_id;
         mdp4_plane->name = pipe_names[pipe_id];
 
-        drm_plane_init(dev, plane, (1 << priv->num_crtcs) - 1, &mdp4_plane_funcs,
-                        mdp4_plane->formats, mdp4_plane->nformats, private_plane);
+        mdp4_plane->nformats = mdp4_get_formats(pipe_id, mdp4_plane->formats,
+                        ARRAY_SIZE(mdp4_plane->formats));
+
+        drm_plane_init(dev, plane, 0xff, &mdp4_plane_funcs,
+                        mdp4_plane->formats, mdp4_plane->nformats,
+                        private_plane);
 
         mdp4_plane_install_properties(plane, &plane->base);
 
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index b3a2f1629041..86537692e45c 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -187,6 +187,7 @@ static int msm_load(struct drm_device *dev, unsigned long flags)
         init_waitqueue_head(&priv->fence_event);
 
         INIT_LIST_HEAD(&priv->inactive_list);
+        INIT_LIST_HEAD(&priv->fence_cbs);
 
         drm_mode_config_init(dev);
 
@@ -539,15 +540,36 @@ int msm_wait_fence_interruptable(struct drm_device *dev, uint32_t fence,
         return ret;
 }
 
-/* call under struct_mutex */
+/* called from workqueue */
 void msm_update_fence(struct drm_device *dev, uint32_t fence)
 {
         struct msm_drm_private *priv = dev->dev_private;
 
-        if (fence > priv->completed_fence) {
-                priv->completed_fence = fence;
-                wake_up_all(&priv->fence_event);
+        mutex_lock(&dev->struct_mutex);
+        priv->completed_fence = max(fence, priv->completed_fence);
+
+        while (!list_empty(&priv->fence_cbs)) {
+                struct msm_fence_cb *cb;
+
+                cb = list_first_entry(&priv->fence_cbs,
+                                struct msm_fence_cb, work.entry);
+
+                if (cb->fence > priv->completed_fence)
+                        break;
+
+                list_del_init(&cb->work.entry);
+                queue_work(priv->wq, &cb->work);
         }
+
+        mutex_unlock(&dev->struct_mutex);
+
+        wake_up_all(&priv->fence_event);
+}
+
+void __msm_fence_worker(struct work_struct *work)
+{
+        struct msm_fence_cb *cb = container_of(work, struct msm_fence_cb, work);
+        cb->func(cb);
 }
 
 /*
@@ -650,13 +672,13 @@ static int msm_ioctl_wait_fence(struct drm_device *dev, void *data,
 }
 
 static const struct drm_ioctl_desc msm_ioctls[] = {
-        DRM_IOCTL_DEF_DRV(MSM_GET_PARAM,    msm_ioctl_get_param,    DRM_UNLOCKED|DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(MSM_GEM_NEW,      msm_ioctl_gem_new,      DRM_UNLOCKED|DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(MSM_GEM_INFO,     msm_ioctl_gem_info,     DRM_UNLOCKED|DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(MSM_GEM_CPU_PREP, msm_ioctl_gem_cpu_prep, DRM_UNLOCKED|DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(MSM_GEM_CPU_FINI, msm_ioctl_gem_cpu_fini, DRM_UNLOCKED|DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(MSM_GEM_SUBMIT,   msm_ioctl_gem_submit,   DRM_UNLOCKED|DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(MSM_WAIT_FENCE,   msm_ioctl_wait_fence,   DRM_UNLOCKED|DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(MSM_GET_PARAM,    msm_ioctl_get_param,    DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(MSM_GEM_NEW,      msm_ioctl_gem_new,      DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(MSM_GEM_INFO,     msm_ioctl_gem_info,     DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(MSM_GEM_CPU_PREP, msm_ioctl_gem_cpu_prep, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(MSM_GEM_CPU_FINI, msm_ioctl_gem_cpu_fini, DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(MSM_GEM_SUBMIT,   msm_ioctl_gem_submit,   DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(MSM_WAIT_FENCE,   msm_ioctl_wait_fence,   DRM_UNLOCKED|DRM_AUTH|DRM_RENDER_ALLOW),
 };
 
 static const struct vm_operations_struct vm_ops = {
@@ -680,7 +702,11 @@ static const struct file_operations fops = {
 };
 
 static struct drm_driver msm_driver = {
-        .driver_features    = DRIVER_HAVE_IRQ | DRIVER_GEM | DRIVER_MODESET,
+        .driver_features    = DRIVER_HAVE_IRQ |
+                                DRIVER_GEM |
+                                DRIVER_PRIME |
+                                DRIVER_RENDER |
+                                DRIVER_MODESET,
         .load               = msm_load,
         .unload             = msm_unload,
         .open               = msm_open,
@@ -698,6 +724,16 @@ static struct drm_driver msm_driver = {
         .dumb_create        = msm_gem_dumb_create,
         .dumb_map_offset    = msm_gem_dumb_map_offset,
         .dumb_destroy       = drm_gem_dumb_destroy,
+        .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+        .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
+        .gem_prime_export   = drm_gem_prime_export,
+        .gem_prime_import   = drm_gem_prime_import,
+        .gem_prime_pin      = msm_gem_prime_pin,
+        .gem_prime_unpin    = msm_gem_prime_unpin,
+        .gem_prime_get_sg_table = msm_gem_prime_get_sg_table,
+        .gem_prime_import_sg_table = msm_gem_prime_import_sg_table,
+        .gem_prime_vmap     = msm_gem_prime_vmap,
+        .gem_prime_vunmap   = msm_gem_prime_vunmap,
 #ifdef CONFIG_DEBUG_FS
         .debugfs_init       = msm_debugfs_init,
         .debugfs_cleanup    = msm_debugfs_cleanup,
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index df8f1d084bc1..d39f0862b19e 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -73,10 +73,16 @@ struct msm_drm_private {
 
         struct workqueue_struct *wq;
 
+        /* callbacks deferred until bo is inactive: */
+        struct list_head fence_cbs;
+
         /* registered IOMMU domains: */
         unsigned int num_iommus;
         struct iommu_domain *iommus[NUM_DOMAINS];
 
+        unsigned int num_planes;
+        struct drm_plane *planes[8];
+
         unsigned int num_crtcs;
         struct drm_crtc *crtcs[8];
 
@@ -94,6 +100,20 @@ struct msm_format {
         uint32_t pixel_format;
 };
 
+/* callback from wq once fence has passed: */
+struct msm_fence_cb {
+        struct work_struct work;
+        uint32_t fence;
+        void (*func)(struct msm_fence_cb *cb);
+};
+
+void __msm_fence_worker(struct work_struct *work);
+
+#define INIT_FENCE_CB(_cb, _func)  do {                     \
+                INIT_WORK(&(_cb)->work, __msm_fence_worker); \
+                (_cb)->func = _func;                         \
+        } while (0)
+
 /* As there are different display controller blocks depending on the
  * snapdragon version, the kms support is split out and the appropriate
  * implementation is loaded at runtime.  The kms module is responsible
@@ -141,17 +161,24 @@ uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj);
 int msm_gem_get_iova_locked(struct drm_gem_object *obj, int id,
                 uint32_t *iova);
 int msm_gem_get_iova(struct drm_gem_object *obj, int id, uint32_t *iova);
+struct page **msm_gem_get_pages(struct drm_gem_object *obj);
+void msm_gem_put_pages(struct drm_gem_object *obj);
 void msm_gem_put_iova(struct drm_gem_object *obj, int id);
 int msm_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
                 struct drm_mode_create_dumb *args);
-int msm_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev,
-                uint32_t handle);
 int msm_gem_dumb_map_offset(struct drm_file *file, struct drm_device *dev,
                 uint32_t handle, uint64_t *offset);
+struct sg_table *msm_gem_prime_get_sg_table(struct drm_gem_object *obj);
+void *msm_gem_prime_vmap(struct drm_gem_object *obj);
+void msm_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);
+struct drm_gem_object *msm_gem_prime_import_sg_table(struct drm_device *dev,
+                size_t size, struct sg_table *sg);
+int msm_gem_prime_pin(struct drm_gem_object *obj);
+void msm_gem_prime_unpin(struct drm_gem_object *obj);
 void *msm_gem_vaddr_locked(struct drm_gem_object *obj);
 void *msm_gem_vaddr(struct drm_gem_object *obj);
-int msm_gem_queue_inactive_work(struct drm_gem_object *obj,
-                struct work_struct *work);
+int msm_gem_queue_inactive_cb(struct drm_gem_object *obj,
+                struct msm_fence_cb *cb);
 void msm_gem_move_to_active(struct drm_gem_object *obj,
                 struct msm_gpu *gpu, bool write, uint32_t fence);
 void msm_gem_move_to_inactive(struct drm_gem_object *obj);
@@ -163,6 +190,8 @@ int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file,
                 uint32_t size, uint32_t flags, uint32_t *handle);
 struct drm_gem_object *msm_gem_new(struct drm_device *dev,
                 uint32_t size, uint32_t flags);
+struct drm_gem_object *msm_gem_import(struct drm_device *dev,
+                uint32_t size, struct sg_table *sgt);
 
 struct drm_gem_object *msm_framebuffer_bo(struct drm_framebuffer *fb, int plane);
 const struct msm_format *msm_framebuffer_format(struct drm_framebuffer *fb);
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index 2bae46c66a30..e587d251c590 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -17,6 +17,7 @@
 
 #include <linux/spinlock.h>
 #include <linux/shmem_fs.h>
+#include <linux/dma-buf.h>
 
 #include "msm_drv.h"
 #include "msm_gem.h"
@@ -77,6 +78,21 @@ static void put_pages(struct drm_gem_object *obj)
         }
 }
 
+struct page **msm_gem_get_pages(struct drm_gem_object *obj)
+{
+        struct drm_device *dev = obj->dev;
+        struct page **p;
+        mutex_lock(&dev->struct_mutex);
+        p = get_pages(obj);
+        mutex_unlock(&dev->struct_mutex);
+        return p;
+}
+
+void msm_gem_put_pages(struct drm_gem_object *obj)
+{
+        /* when we start tracking the pin count, then do something here */
+}
+
 int msm_gem_mmap_obj(struct drm_gem_object *obj,
                 struct vm_area_struct *vma)
 {
@@ -162,6 +178,11 @@ out:
         case 0:
         case -ERESTARTSYS:
         case -EINTR:
+        case -EBUSY:
+                /*
+                 * EBUSY is ok: this just means that another thread
+                 * already did the job.
+                 */
                 return VM_FAULT_NOPAGE;
         case -ENOMEM:
                 return VM_FAULT_OOM;
@@ -293,7 +314,17 @@ int msm_gem_get_iova_locked(struct drm_gem_object *obj, int id,
 
 int msm_gem_get_iova(struct drm_gem_object *obj, int id, uint32_t *iova)
 {
+        struct msm_gem_object *msm_obj = to_msm_bo(obj);
         int ret;
+
+        /* this is safe right now because we don't unmap until the
+         * bo is deleted:
+         */
+        if (msm_obj->domain[id].iova) {
+                *iova = msm_obj->domain[id].iova;
+                return 0;
+        }
+
         mutex_lock(&obj->dev->struct_mutex);
         ret = msm_gem_get_iova_locked(obj, id, iova);
         mutex_unlock(&obj->dev->struct_mutex);
@@ -363,8 +394,11 @@ void *msm_gem_vaddr(struct drm_gem_object *obj)
         return ret;
 }
 
-int msm_gem_queue_inactive_work(struct drm_gem_object *obj,
-                struct work_struct *work)
+/* setup callback for when bo is no longer busy..
+ * TODO probably want to differentiate read vs write..
+ */
+int msm_gem_queue_inactive_cb(struct drm_gem_object *obj,
+                struct msm_fence_cb *cb)
 {
         struct drm_device *dev = obj->dev;
         struct msm_drm_private *priv = dev->dev_private;
@@ -372,12 +406,13 @@ int msm_gem_queue_inactive_work(struct drm_gem_object *obj,
         int ret = 0;
 
         mutex_lock(&dev->struct_mutex);
-        if (!list_empty(&work->entry)) {
+        if (!list_empty(&cb->work.entry)) {
                 ret = -EINVAL;
         } else if (is_active(msm_obj)) {
-                list_add_tail(&work->entry, &msm_obj->inactive_work);
+                cb->fence = max(msm_obj->read_fence, msm_obj->write_fence);
+                list_add_tail(&cb->work.entry, &priv->fence_cbs);
         } else {
-                queue_work(priv->wq, work);
+                queue_work(priv->wq, &cb->work);
         }
         mutex_unlock(&dev->struct_mutex);
 
@@ -410,16 +445,6 @@ void msm_gem_move_to_inactive(struct drm_gem_object *obj)
         msm_obj->write_fence = 0;
         list_del_init(&msm_obj->mm_list);
         list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
-
-        while (!list_empty(&msm_obj->inactive_work)) {
-                struct work_struct *work;
-
-                work = list_first_entry(&msm_obj->inactive_work,
-                                struct work_struct, entry);
-
-                list_del_init(&work->entry);
-                queue_work(priv->wq, work);
-        }
 }
 
 int msm_gem_cpu_prep(struct drm_gem_object *obj, uint32_t op,
@@ -510,10 +535,21 @@ void msm_gem_free_object(struct drm_gem_object *obj)
 
         drm_gem_free_mmap_offset(obj);
 
-        if (msm_obj->vaddr)
-                vunmap(msm_obj->vaddr);
+        if (obj->import_attach) {
+                if (msm_obj->vaddr)
+                        dma_buf_vunmap(obj->import_attach->dmabuf, msm_obj->vaddr);
 
-        put_pages(obj);
+                /* Don't drop the pages for imported dmabuf, as they are not
+                 * ours, just free the array we allocated:
+                 */
+                if (msm_obj->pages)
+                        drm_free_large(msm_obj->pages);
+
+        } else {
+                if (msm_obj->vaddr)
+                        vunmap(msm_obj->vaddr);
+                put_pages(obj);
+        }
 
         if (msm_obj->resv == &msm_obj->_resv)
                 reservation_object_fini(msm_obj->resv);
@@ -549,17 +585,12 @@ int msm_gem_new_handle(struct drm_device *dev, struct drm_file *file,
         return ret;
 }
 
-struct drm_gem_object *msm_gem_new(struct drm_device *dev,
-                uint32_t size, uint32_t flags)
+static int msm_gem_new_impl(struct drm_device *dev,
+                uint32_t size, uint32_t flags,
+                struct drm_gem_object **obj)
 {
         struct msm_drm_private *priv = dev->dev_private;
         struct msm_gem_object *msm_obj;
-        struct drm_gem_object *obj = NULL;
-        int ret;
-
-        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
-
-        size = PAGE_ALIGN(size);
 
         switch (flags & MSM_BO_CACHE_MASK) {
         case MSM_BO_UNCACHED:
@@ -569,21 +600,12 @@ struct drm_gem_object *msm_gem_new(struct drm_device *dev,
         default:
                 dev_err(dev->dev, "invalid cache flag: %x\n",
                                 (flags & MSM_BO_CACHE_MASK));
-                ret = -EINVAL;
-                goto fail;
+                return -EINVAL;
         }
 
         msm_obj = kzalloc(sizeof(*msm_obj), GFP_KERNEL);
-        if (!msm_obj) {
-                ret = -ENOMEM;
-                goto fail;
-        }
-
-        obj = &msm_obj->base;
-
-        ret = drm_gem_object_init(dev, obj, size);
-        if (ret)
-                goto fail;
+        if (!msm_obj)
+                return -ENOMEM;
 
         msm_obj->flags = flags;
 
@@ -591,9 +613,69 @@ struct drm_gem_object *msm_gem_new(struct drm_device *dev,
         reservation_object_init(msm_obj->resv);
 
         INIT_LIST_HEAD(&msm_obj->submit_entry);
-        INIT_LIST_HEAD(&msm_obj->inactive_work);
         list_add_tail(&msm_obj->mm_list, &priv->inactive_list);
 
+        *obj = &msm_obj->base;
+
+        return 0;
+}
+
+struct drm_gem_object *msm_gem_new(struct drm_device *dev,
+                uint32_t size, uint32_t flags)
+{
+        struct drm_gem_object *obj;
+        int ret;
+
+        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+
+        size = PAGE_ALIGN(size);
+
+        ret = msm_gem_new_impl(dev, size, flags, &obj);
+        if (ret)
+                goto fail;
+
+        ret = drm_gem_object_init(dev, obj, size);
+        if (ret)
+                goto fail;
+
+        return obj;
+
+fail:
+        if (obj)
+                drm_gem_object_unreference_unlocked(obj);
+
+        return ERR_PTR(ret);
+}
+
+struct drm_gem_object *msm_gem_import(struct drm_device *dev,
+                uint32_t size, struct sg_table *sgt)
+{
+        struct msm_gem_object *msm_obj;
+        struct drm_gem_object *obj;
+        int ret, npages;
+
+        size = PAGE_ALIGN(size);
+
+        ret = msm_gem_new_impl(dev, size, MSM_BO_WC, &obj);
+        if (ret)
+                goto fail;
+
+        drm_gem_private_object_init(dev, obj, size);
+
+        npages = size / PAGE_SIZE;
+
+        msm_obj = to_msm_bo(obj);
+        msm_obj->sgt = sgt;
+        msm_obj->pages = drm_malloc_ab(npages, sizeof(struct page *));
+        if (!msm_obj->pages) {
+                ret = -ENOMEM;
+                goto fail;
+        }
+
+        ret = drm_prime_sg_to_page_addr_arrays(sgt, msm_obj->pages, NULL, npages);
+        if (ret)
+                goto fail;
+
         return obj;
 
 fail:
diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h
index 0676f32e2c6a..f4f23a578d9d 100644
--- a/drivers/gpu/drm/msm/msm_gem.h
+++ b/drivers/gpu/drm/msm/msm_gem.h
@@ -45,9 +45,6 @@ struct msm_gem_object {
          */
         struct list_head submit_entry;
 
-        /* work defered until bo is inactive: */
-        struct list_head inactive_work;
-
         struct page **pages;
         struct sg_table *sgt;
         void *vaddr;
diff --git a/drivers/gpu/drm/msm/msm_gem_prime.c b/drivers/gpu/drm/msm/msm_gem_prime.c
new file mode 100644
index 000000000000..d48f9fc5129b
--- /dev/null
+++ b/drivers/gpu/drm/msm/msm_gem_prime.c
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2013 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "msm_drv.h"
+#include "msm_gem.h"
+
+
+struct sg_table *msm_gem_prime_get_sg_table(struct drm_gem_object *obj)
+{
+        struct msm_gem_object *msm_obj = to_msm_bo(obj);
+        BUG_ON(!msm_obj->sgt);  /* should have already pinned! */
+        return msm_obj->sgt;
+}
+
+void *msm_gem_prime_vmap(struct drm_gem_object *obj)
+{
+        return msm_gem_vaddr(obj);
+}
+
+void msm_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
+{
+        /* TODO msm_gem_vunmap() */
+}
+
+struct drm_gem_object *msm_gem_prime_import_sg_table(struct drm_device *dev,
+                size_t size, struct sg_table *sg)
+{
+        return msm_gem_import(dev, size, sg);
+}
+
+int msm_gem_prime_pin(struct drm_gem_object *obj)
+{
+        if (!obj->import_attach)
+                msm_gem_get_pages(obj);
+        return 0;
+}
+
+void msm_gem_prime_unpin(struct drm_gem_object *obj)
+{
+        if (!obj->import_attach)
+                msm_gem_put_pages(obj);
+}
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 3bab937965d1..4583d61556f5 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -268,6 +268,8 @@ static void retire_worker(struct work_struct *work)
         struct drm_device *dev = gpu->dev;
         uint32_t fence = gpu->funcs->last_fence(gpu);
 
+        msm_update_fence(gpu->dev, fence);
+
         mutex_lock(&dev->struct_mutex);
 
         while (!list_empty(&gpu->active_list)) {
@@ -287,8 +289,6 @@ static void retire_worker(struct work_struct *work)
                 }
         }
 
-        msm_update_fence(gpu->dev, fence);
-
         mutex_unlock(&dev->struct_mutex);
 }
 
diff --git a/drivers/gpu/drm/nouveau/Makefile b/drivers/gpu/drm/nouveau/Makefile
index d939a1da3203..edcf801613e6 100644
--- a/drivers/gpu/drm/nouveau/Makefile
+++ b/drivers/gpu/drm/nouveau/Makefile
@@ -28,7 +28,9 @@ nouveau-y += core/subdev/bar/nv50.o
 nouveau-y += core/subdev/bar/nvc0.o
 nouveau-y += core/subdev/bios/base.o
 nouveau-y += core/subdev/bios/bit.o
+nouveau-y += core/subdev/bios/boost.o
 nouveau-y += core/subdev/bios/conn.o
+nouveau-y += core/subdev/bios/cstep.o
 nouveau-y += core/subdev/bios/dcb.o
 nouveau-y += core/subdev/bios/disp.o
 nouveau-y += core/subdev/bios/dp.o
@@ -39,17 +41,26 @@ nouveau-y += core/subdev/bios/init.o
 nouveau-y += core/subdev/bios/mxm.o
 nouveau-y += core/subdev/bios/perf.o
 nouveau-y += core/subdev/bios/pll.o
+nouveau-y += core/subdev/bios/rammap.o
+nouveau-y += core/subdev/bios/timing.o
 nouveau-y += core/subdev/bios/therm.o
+nouveau-y += core/subdev/bios/vmap.o
+nouveau-y += core/subdev/bios/volt.o
 nouveau-y += core/subdev/bios/xpio.o
+nouveau-y += core/subdev/bus/hwsq.o
 nouveau-y += core/subdev/bus/nv04.o
 nouveau-y += core/subdev/bus/nv31.o
 nouveau-y += core/subdev/bus/nv50.o
+nouveau-y += core/subdev/bus/nv94.o
 nouveau-y += core/subdev/bus/nvc0.o
+nouveau-y += core/subdev/clock/base.o
 nouveau-y += core/subdev/clock/nv04.o
 nouveau-y += core/subdev/clock/nv40.o
 nouveau-y += core/subdev/clock/nv50.o
+nouveau-y += core/subdev/clock/nv84.o
 nouveau-y += core/subdev/clock/nva3.o
 nouveau-y += core/subdev/clock/nvc0.o
+nouveau-y += core/subdev/clock/nve0.o
 nouveau-y += core/subdev/clock/pllnv04.o
 nouveau-y += core/subdev/clock/pllnva3.o
 nouveau-y += core/subdev/devinit/base.o
@@ -78,7 +89,12 @@ nouveau-y += core/subdev/fb/nv47.o
 nouveau-y += core/subdev/fb/nv49.o
 nouveau-y += core/subdev/fb/nv4e.o
 nouveau-y += core/subdev/fb/nv50.o
+nouveau-y += core/subdev/fb/nv84.o
+nouveau-y += core/subdev/fb/nva3.o
+nouveau-y += core/subdev/fb/nvaa.o
+nouveau-y += core/subdev/fb/nvaf.o
 nouveau-y += core/subdev/fb/nvc0.o
+nouveau-y += core/subdev/fb/nve0.o
 nouveau-y += core/subdev/fb/ramnv04.o
 nouveau-y += core/subdev/fb/ramnv10.o
 nouveau-y += core/subdev/fb/ramnv1a.o
@@ -89,7 +105,12 @@ nouveau-y += core/subdev/fb/ramnv44.o
 nouveau-y += core/subdev/fb/ramnv49.o
 nouveau-y += core/subdev/fb/ramnv4e.o
 nouveau-y += core/subdev/fb/ramnv50.o
+nouveau-y += core/subdev/fb/ramnva3.o
+nouveau-y += core/subdev/fb/ramnvaa.o
 nouveau-y += core/subdev/fb/ramnvc0.o
+nouveau-y += core/subdev/fb/ramnve0.o
+nouveau-y += core/subdev/fb/sddr3.o
+nouveau-y += core/subdev/fb/gddr5.o
 nouveau-y += core/subdev/gpio/base.o
 nouveau-y += core/subdev/gpio/nv10.o
 nouveau-y += core/subdev/gpio/nv50.o
@@ -113,13 +134,22 @@ nouveau-y += core/subdev/instmem/nv50.o
 nouveau-y += core/subdev/ltcg/nvc0.o
 nouveau-y += core/subdev/mc/base.o
 nouveau-y += core/subdev/mc/nv04.o
+nouveau-y += core/subdev/mc/nv40.o
 nouveau-y += core/subdev/mc/nv44.o
 nouveau-y += core/subdev/mc/nv50.o
+nouveau-y += core/subdev/mc/nv94.o
 nouveau-y += core/subdev/mc/nv98.o
 nouveau-y += core/subdev/mc/nvc0.o
+nouveau-y += core/subdev/mc/nvc3.o
 nouveau-y += core/subdev/mxm/base.o
 nouveau-y += core/subdev/mxm/mxms.o
 nouveau-y += core/subdev/mxm/nv50.o
+nouveau-y += core/subdev/pwr/base.o
+nouveau-y += core/subdev/pwr/memx.o
+nouveau-y += core/subdev/pwr/nva3.o
+nouveau-y += core/subdev/pwr/nvc0.o
+nouveau-y += core/subdev/pwr/nvd0.o
+nouveau-y += core/subdev/pwr/nv108.o
 nouveau-y += core/subdev/therm/base.o
 nouveau-y += core/subdev/therm/fan.o
 nouveau-y += core/subdev/therm/fannil.o
@@ -140,6 +170,9 @@ nouveau-y += core/subdev/vm/nv41.o
 nouveau-y += core/subdev/vm/nv44.o
 nouveau-y += core/subdev/vm/nv50.o
 nouveau-y += core/subdev/vm/nvc0.o
+nouveau-y += core/subdev/volt/base.o
+nouveau-y += core/subdev/volt/gpio.o
+nouveau-y += core/subdev/volt/nv40.o
 
 nouveau-y += core/engine/falcon.o
 nouveau-y += core/engine/xtensa.o
@@ -158,6 +191,7 @@ nouveau-y += core/engine/copy/nve0.o
 nouveau-y += core/engine/crypt/nv84.o
 nouveau-y += core/engine/crypt/nv98.o
 nouveau-y += core/engine/device/base.o
+nouveau-y += core/engine/device/ctrl.o
 nouveau-y += core/engine/device/nv04.o
 nouveau-y += core/engine/device/nv10.o
 nouveau-y += core/engine/device/nv20.o
@@ -227,8 +261,18 @@ nouveau-y += core/engine/graph/nve4.o
 nouveau-y += core/engine/graph/nvf0.o
 nouveau-y += core/engine/mpeg/nv31.o
 nouveau-y += core/engine/mpeg/nv40.o
+nouveau-y += core/engine/mpeg/nv44.o
 nouveau-y += core/engine/mpeg/nv50.o
 nouveau-y += core/engine/mpeg/nv84.o
+nouveau-y += core/engine/perfmon/base.o
+nouveau-y += core/engine/perfmon/daemon.o
+nouveau-y += core/engine/perfmon/nv40.o
+nouveau-y += core/engine/perfmon/nv50.o
+nouveau-y += core/engine/perfmon/nv84.o
+nouveau-y += core/engine/perfmon/nva3.o
+nouveau-y += core/engine/perfmon/nvc0.o
+nouveau-y += core/engine/perfmon/nve0.o
+nouveau-y += core/engine/perfmon/nvf0.o
 nouveau-y += core/engine/ppp/nv98.o
 nouveau-y += core/engine/ppp/nvc0.o
 nouveau-y += core/engine/software/nv04.o
@@ -260,9 +304,7 @@ include $(src)/dispnv04/Makefile
 nouveau-y += nv50_display.o
 
 # drm/pm
-nouveau-y += nouveau_pm.o nouveau_volt.o nouveau_perf.o
-nouveau-y += nv04_pm.o nv40_pm.o nv50_pm.o nva3_pm.o nvc0_pm.o
-nouveau-y += nouveau_mem.o
+nouveau-y += nouveau_hwmon.o nouveau_sysfs.o
 
 # other random bits
 nouveau-$(CONFIG_COMPAT) += nouveau_ioc32.o
diff --git a/drivers/gpu/drm/nouveau/core/core/event.c b/drivers/gpu/drm/nouveau/core/core/event.c
index 7eb81c1b6fab..3f3c76581a9e 100644
--- a/drivers/gpu/drm/nouveau/core/core/event.c
+++ b/drivers/gpu/drm/nouveau/core/core/event.c
@@ -23,62 +23,114 @@
 #include <core/os.h>
 #include <core/event.h>
 
-static void
-nouveau_event_put_locked(struct nouveau_event *event, int index,
-                         struct nouveau_eventh *handler)
+void
+nouveau_event_put(struct nouveau_eventh *handler)
 {
-        if (!--event->index[index].refs) {
-                if (event->disable)
-                        event->disable(event, index);
+        struct nouveau_event *event = handler->event;
+        unsigned long flags;
+        if (__test_and_clear_bit(NVKM_EVENT_ENABLE, &handler->flags)) {
+                spin_lock_irqsave(&event->refs_lock, flags);
+                if (!--event->index[handler->index].refs) {
+                        if (event->disable)
+                                event->disable(event, handler->index);
+                }
+                spin_unlock_irqrestore(&event->refs_lock, flags);
         }
-        list_del(&handler->head);
 }
 
 void
-nouveau_event_put(struct nouveau_event *event, int index,
-                  struct nouveau_eventh *handler)
+nouveau_event_get(struct nouveau_eventh *handler)
 {
+        struct nouveau_event *event = handler->event;
         unsigned long flags;
+        if (!__test_and_set_bit(NVKM_EVENT_ENABLE, &handler->flags)) {
+                spin_lock_irqsave(&event->refs_lock, flags);
+                if (!event->index[handler->index].refs++) {
+                        if (event->enable)
+                                event->enable(event, handler->index);
+                }
+                spin_unlock_irqrestore(&event->refs_lock, flags);
+        }
+}
 
-        spin_lock_irqsave(&event->lock, flags);
-        if (index < event->index_nr)
-                nouveau_event_put_locked(event, index, handler);
-        spin_unlock_irqrestore(&event->lock, flags);
+static void
+nouveau_event_fini(struct nouveau_eventh *handler)
+{
+        struct nouveau_event *event = handler->event;
+        unsigned long flags;
+        nouveau_event_put(handler);
+        spin_lock_irqsave(&event->list_lock, flags);
+        list_del(&handler->head);
+        spin_unlock_irqrestore(&event->list_lock, flags);
 }
 
-void
-nouveau_event_get(struct nouveau_event *event, int index,
-                  struct nouveau_eventh *handler)
+static int
+nouveau_event_init(struct nouveau_event *event, int index,
+                   int (*func)(void *, int), void *priv,
+                   struct nouveau_eventh *handler)
 {
         unsigned long flags;
 
-        spin_lock_irqsave(&event->lock, flags);
-        if (index < event->index_nr) {
-                list_add(&handler->head, &event->index[index].list);
-                if (!event->index[index].refs++) {
-                        if (event->enable)
-                                event->enable(event, index);
-                }
+        if (index >= event->index_nr)
+                return -EINVAL;
+
+        handler->event = event;
+        handler->flags = 0;
+        handler->index = index;
+        handler->func = func;
+        handler->priv = priv;
+
+        spin_lock_irqsave(&event->list_lock, flags);
+        list_add_tail(&handler->head, &event->index[index].list);
+        spin_unlock_irqrestore(&event->list_lock, flags);
+        return 0;
+}
+
+int
+nouveau_event_new(struct nouveau_event *event, int index,
+                  int (*func)(void *, int), void *priv,
+                  struct nouveau_eventh **phandler)
+{
+        struct nouveau_eventh *handler;
+        int ret = -ENOMEM;
+
+        handler = *phandler = kmalloc(sizeof(*handler), GFP_KERNEL);
+        if (handler) {
+                ret = nouveau_event_init(event, index, func, priv, handler);
+                if (ret)
+                        kfree(handler);
         }
-        spin_unlock_irqrestore(&event->lock, flags);
+
+        return ret;
+}
+
+void
+nouveau_event_ref(struct nouveau_eventh *handler, struct nouveau_eventh **ref)
+{
+        BUG_ON(handler != NULL);
+        if (*ref) {
+                nouveau_event_fini(*ref);
+                kfree(*ref);
+        }
+        *ref = handler;
 }
 
 void
 nouveau_event_trigger(struct nouveau_event *event, int index)
 {
-        struct nouveau_eventh *handler, *temp;
+        struct nouveau_eventh *handler;
         unsigned long flags;
 
-        if (index >= event->index_nr)
+        if (WARN_ON(index >= event->index_nr))
                 return;
 
-        spin_lock_irqsave(&event->lock, flags);
-        list_for_each_entry_safe(handler, temp, &event->index[index].list, head) {
-                if (handler->func(handler, index) == NVKM_EVENT_DROP) {
-                        nouveau_event_put_locked(event, index, handler);
-                }
+        spin_lock_irqsave(&event->list_lock, flags);
+        list_for_each_entry(handler, &event->index[index].list, head) {
+                if (test_bit(NVKM_EVENT_ENABLE, &handler->flags) &&
+                    handler->func(handler->priv, index) == NVKM_EVENT_DROP)
+                        nouveau_event_put(handler);
         }
-        spin_unlock_irqrestore(&event->lock, flags);
+        spin_unlock_irqrestore(&event->list_lock, flags);
 }
 
 void
@@ -102,7 +154,8 @@ nouveau_event_create(int index_nr, struct nouveau_event **pevent)
         if (!event)
                 return -ENOMEM;
 
-        spin_lock_init(&event->lock);
+        spin_lock_init(&event->list_lock);
+        spin_lock_init(&event->refs_lock);
         for (i = 0; i < index_nr; i++)
                 INIT_LIST_HEAD(&event->index[i].list);
         event->index_nr = index_nr;
diff --git a/drivers/gpu/drm/nouveau/core/core/option.c b/drivers/gpu/drm/nouveau/core/core/option.c
index 62a432ea39e5..9f6fcc5f66c2 100644
--- a/drivers/gpu/drm/nouveau/core/core/option.c
+++ b/drivers/gpu/drm/nouveau/core/core/option.c
@@ -25,15 +25,6 @@
 #include <core/option.h>
 #include <core/debug.h>
 
-/* compares unterminated string 'str' with zero-terminated string 'cmp' */
-static inline int
-strncasecmpz(const char *str, const char *cmp, size_t len)
-{
-        if (strlen(cmp) != len)
-                return len;
-        return strncasecmp(str, cmp, len);
-}
-
 const char *
 nouveau_stropt(const char *optstr, const char *opt, int *arglen)
 {
@@ -105,7 +96,7 @@ nouveau_dbgopt(const char *optstr, const char *sub)
                                 else if (!strncasecmpz(optstr, "warn", len))
                                         level = NV_DBG_WARN;
                                 else if (!strncasecmpz(optstr, "info", len))
-                                        level = NV_DBG_INFO;
+                                        level = NV_DBG_INFO_NORMAL;
                                 else if (!strncasecmpz(optstr, "debug", len))
                                         level = NV_DBG_DEBUG;
                                 else if (!strncasecmpz(optstr, "trace", len))
diff --git a/drivers/gpu/drm/nouveau/core/core/printk.c b/drivers/gpu/drm/nouveau/core/core/printk.c
index 52fb2aa129e8..03e0060b13da 100644
--- a/drivers/gpu/drm/nouveau/core/core/printk.c
+++ b/drivers/gpu/drm/nouveau/core/core/printk.c
@@ -27,16 +27,38 @@
 #include <core/subdev.h>
 #include <core/printk.h>
 
-int nv_printk_suspend_level = NV_DBG_DEBUG;
+int nv_info_debug_level = NV_DBG_INFO_NORMAL;
 
 void
-nv_printk_(struct nouveau_object *object, const char *pfx, int level,
-           const char *fmt, ...)
+nv_printk_(struct nouveau_object *object, int level, const char *fmt, ...)
 {
         static const char name[] = { '!', 'E', 'W', ' ', 'D', 'T', 'P', 'S' };
+        const char *pfx;
         char mfmt[256];
         va_list args;
 
+        switch (level) {
+        case NV_DBG_FATAL:
+                pfx = KERN_CRIT;
+                break;
+        case NV_DBG_ERROR:
+                pfx = KERN_ERR;
+                break;
+        case NV_DBG_WARN:
+                pfx = KERN_WARNING;
+                break;
+        case NV_DBG_INFO_NORMAL:
+                pfx = KERN_INFO;
+                break;
+        case NV_DBG_DEBUG:
+        case NV_DBG_PARANOIA:
+        case NV_DBG_TRACE:
+        case NV_DBG_SPAM:
+        default:
+                pfx = KERN_DEBUG;
+                break;
+        }
+
         if (object && !nv_iclass(object, NV_CLIENT_CLASS)) {
                 struct nouveau_object *device = object;
                 struct nouveau_object *subdev = object;
@@ -74,20 +96,3 @@ nv_printk_(struct nouveau_object *object, const char *pfx, int level,
         vprintk(mfmt, args);
         va_end(args);
 }
-
-#define CONV_LEVEL(x) case NV_DBG_##x: return NV_PRINTK_##x
-
-const char *nv_printk_level_to_pfx(int level)
-{
-        switch (level) {
-        CONV_LEVEL(FATAL);
-        CONV_LEVEL(ERROR);
-        CONV_LEVEL(WARN);
-        CONV_LEVEL(INFO);
-        CONV_LEVEL(DEBUG);
-        CONV_LEVEL(PARANOIA);
-        CONV_LEVEL(TRACE);
-        CONV_LEVEL(SPAM);
-        }
-        return NV_PRINTK_DEBUG;
-}
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/base.c b/drivers/gpu/drm/nouveau/core/engine/device/base.c
index 4c72571655ad..9135b25a29d0 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/base.c
@@ -29,7 +29,7 @@
 
 #include <core/class.h>
 
-#include <engine/device.h>
+#include "priv.h"
 
 static DEFINE_MUTEX(nv_devices_mutex);
 static LIST_HEAD(nv_devices);
@@ -75,7 +75,9 @@ static const u64 disable_map[] = {
         [NVDEV_SUBDEV_BAR]      = NV_DEVICE_DISABLE_CORE,
         [NVDEV_SUBDEV_VOLT]     = NV_DEVICE_DISABLE_CORE,
         [NVDEV_SUBDEV_THERM]    = NV_DEVICE_DISABLE_CORE,
+        [NVDEV_SUBDEV_PWR]      = NV_DEVICE_DISABLE_CORE,
         [NVDEV_ENGINE_DMAOBJ]   = NV_DEVICE_DISABLE_CORE,
+        [NVDEV_ENGINE_PERFMON]  = NV_DEVICE_DISABLE_CORE,
         [NVDEV_ENGINE_FIFO]     = NV_DEVICE_DISABLE_FIFO,
         [NVDEV_ENGINE_SW]       = NV_DEVICE_DISABLE_FIFO,
         [NVDEV_ENGINE_GR]       = NV_DEVICE_DISABLE_GRAPH,
@@ -87,7 +89,7 @@ static const u64 disable_map[] = {
         [NVDEV_ENGINE_PPP]      = NV_DEVICE_DISABLE_PPP,
         [NVDEV_ENGINE_COPY0]    = NV_DEVICE_DISABLE_COPY0,
         [NVDEV_ENGINE_COPY1]    = NV_DEVICE_DISABLE_COPY1,
-        [NVDEV_ENGINE_UNK1C1]   = NV_DEVICE_DISABLE_UNK1C1,
+        [NVDEV_ENGINE_VIC]      = NV_DEVICE_DISABLE_VIC,
         [NVDEV_ENGINE_VENC]     = NV_DEVICE_DISABLE_VENC,
         [NVDEV_ENGINE_DISP]     = NV_DEVICE_DISABLE_DISP,
         [NVDEV_SUBDEV_NR]       = 0,
@@ -119,10 +121,12 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
                         return -ENODEV;
         }
 
-        ret = nouveau_parent_create(parent, nv_object(device), oclass, 0, NULL,
+        ret = nouveau_parent_create(parent, nv_object(device), oclass, 0,
+                                    nouveau_control_oclass,
                                     (1ULL << NVDEV_ENGINE_DMAOBJ) |
                                     (1ULL << NVDEV_ENGINE_FIFO) |
-                                    (1ULL << NVDEV_ENGINE_DISP), &devobj);
+                                    (1ULL << NVDEV_ENGINE_DISP) |
+                                    (1ULL << NVDEV_ENGINE_PERFMON), &devobj);
         *pobject = nv_object(devobj);
         if (ret)
                 return ret;
@@ -158,22 +162,29 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
                 iounmap(map);
 
                 /* determine chipset and derive architecture from it */
-                if ((boot0 & 0x0f000000) > 0) {
-                        device->chipset = (boot0 & 0xff00000) >> 20;
-                        switch (device->chipset & 0xf0) {
-                        case 0x10: device->card_type = NV_10; break;
-                        case 0x20: device->card_type = NV_20; break;
-                        case 0x30: device->card_type = NV_30; break;
-                        case 0x40:
-                        case 0x60: device->card_type = NV_40; break;
-                        case 0x50:
-                        case 0x80:
-                        case 0x90:
-                        case 0xa0: device->card_type = NV_50; break;
-                        case 0xc0: device->card_type = NV_C0; break;
-                        case 0xd0: device->card_type = NV_D0; break;
-                        case 0xe0:
-                        case 0xf0: device->card_type = NV_E0; break;
+                if ((boot0 & 0x1f000000) > 0) {
+                        device->chipset = (boot0 & 0x1ff00000) >> 20;
+                        switch (device->chipset & 0x1f0) {
+                        case 0x010: {
+                                if (0x461 & (1 << (device->chipset & 0xf)))
+                                        device->card_type = NV_10;
+                                else
+                                        device->card_type = NV_11;
+                                break;
+                        }
+                        case 0x020: device->card_type = NV_20; break;
+                        case 0x030: device->card_type = NV_30; break;
+                        case 0x040:
+                        case 0x060: device->card_type = NV_40; break;
+                        case 0x050:
+                        case 0x080:
+                        case 0x090:
+                        case 0x0a0: device->card_type = NV_50; break;
+                        case 0x0c0: device->card_type = NV_C0; break;
+                        case 0x0d0: device->card_type = NV_D0; break;
+                        case 0x0e0:
+                        case 0x0f0:
+                        case 0x100: device->card_type = NV_E0; break;
                         default:
                                 break;
                         }
@@ -188,7 +199,8 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
 
                 switch (device->card_type) {
                 case NV_04: ret = nv04_identify(device); break;
-                case NV_10: ret = nv10_identify(device); break;
+                case NV_10:
+                case NV_11: ret = nv10_identify(device); break;
                 case NV_20: ret = nv20_identify(device); break;
                 case NV_30: ret = nv30_identify(device); break;
                 case NV_40: ret = nv40_identify(device); break;
@@ -212,7 +224,7 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
                 nv_info(device, "Family : NV%02X\n", device->card_type);
 
                 /* determine frequency of timing crystal */
-                if ( device->chipset < 0x17 ||
+                if ( device->card_type <= NV_10 || device->chipset < 0x17 ||
                     (device->chipset >= 0x20 && device->chipset < 0x25))
                         strap &= 0x00000040;
                 else
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/ctrl.c b/drivers/gpu/drm/nouveau/core/engine/device/ctrl.c
new file mode 100644
index 000000000000..4b69bf56ed01
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/device/ctrl.c
@@ -0,0 +1,144 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include <core/object.h>
+#include <core/class.h>
+
+#include <subdev/clock.h>
+
+#include "priv.h"
+
+static int
+nouveau_control_mthd_pstate_info(struct nouveau_object *object, u32 mthd,
+                                void *data, u32 size)
+{
+        struct nouveau_clock *clk = nouveau_clock(object);
+        struct nv_control_pstate_info *args = data;
+
+        if (size < sizeof(*args))
+                return -EINVAL;
+
+        if (clk) {
+                args->count  = clk->state_nr;
+                args->ustate = clk->ustate;
+                args->pstate = clk->pstate;
+        } else {
+                args->count  = 0;
+                args->ustate = NV_CONTROL_PSTATE_INFO_USTATE_DISABLE;
+                args->pstate = NV_CONTROL_PSTATE_INFO_PSTATE_UNKNOWN;
+        }
+
+        return 0;
+}
+
+static int
+nouveau_control_mthd_pstate_attr(struct nouveau_object *object, u32 mthd,
+                                void *data, u32 size)
+{
+        struct nouveau_clock *clk = nouveau_clock(object);
+        struct nv_control_pstate_attr *args = data;
+        struct nouveau_clocks *domain;
+        struct nouveau_pstate *pstate;
+        struct nouveau_cstate *cstate;
+        int i = 0, j = -1;
+        u32 lo, hi;
+
+        if ((size < sizeof(*args)) || !clk ||
+            (args->state >= 0 && args->state >= clk->state_nr))
+                return -EINVAL;
+        domain = clk->domains;
+
+        while (domain->name != nv_clk_src_max) {
+                if (domain->mname && ++j == args->index)
+                        break;
+                domain++;
+        }
+
+        if (domain->name == nv_clk_src_max)
+                return -EINVAL;
+
+        if (args->state != NV_CONTROL_PSTATE_ATTR_STATE_CURRENT) {
+                list_for_each_entry(pstate, &clk->states, head) {
+                        if (i++ == args->state)
+                                break;
+                }
+
+                lo = pstate->base.domain[domain->name];
+                hi = lo;
+                list_for_each_entry(cstate, &pstate->list, head) {
+                        lo = min(lo, cstate->domain[domain->name]);
+                        hi = max(hi, cstate->domain[domain->name]);
+                }
+
+                args->state = pstate->pstate;
+        } else {
+                lo = max(clk->read(clk, domain->name), 0);
+                hi = lo;
+        }
+
+        snprintf(args->name, sizeof(args->name), "%s", domain->mname);
+        snprintf(args->unit, sizeof(args->unit), "MHz");
+        args->min = lo / domain->mdiv;
+        args->max = hi / domain->mdiv;
+
+        args->index = 0;
+        while ((++domain)->name != nv_clk_src_max) {
+                if (domain->mname) {
+                        args->index = ++j;
+                        break;
+                }
+        }
+
+        return 0;
+}
+
+static int
+nouveau_control_mthd_pstate_user(struct nouveau_object *object, u32 mthd,
+                                void *data, u32 size)
+{
+        struct nouveau_clock *clk = nouveau_clock(object);
+        struct nv_control_pstate_user *args = data;
+
+        if (size < sizeof(*args) || !clk)
+                return -EINVAL;
+
+        return nouveau_clock_ustate(clk, args->state);
+}
+
+struct nouveau_oclass
+nouveau_control_oclass[] = {
+        { .handle = NV_CONTROL_CLASS,
+          .ofuncs = &nouveau_object_ofuncs,
+          .omthds = (struct nouveau_omthds[]) {
+                  { NV_CONTROL_PSTATE_INFO,
+                    NV_CONTROL_PSTATE_INFO, nouveau_control_mthd_pstate_info },
+                  { NV_CONTROL_PSTATE_ATTR,
+                    NV_CONTROL_PSTATE_ATTR, nouveau_control_mthd_pstate_attr },
+                  { NV_CONTROL_PSTATE_USER,
+                    NV_CONTROL_PSTATE_USER, nouveau_control_mthd_pstate_user },
+                  {},
+          },
+        },
+        {}
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv04.c b/drivers/gpu/drm/nouveau/core/engine/device/nv04.c
index a0284cf09c0f..dbd2dde7b7e7 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv04.c
@@ -50,15 +50,15 @@ nv04_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv04_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv04_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv04_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv04_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv04_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv04_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv04_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv04_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
@@ -68,15 +68,15 @@ nv04_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv05_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv04_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv04_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv04_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv04_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv04_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv04_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv04_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv10.c b/drivers/gpu/drm/nouveau/core/engine/device/nv10.c
index 1b7809a095c3..6e03dd6abeea 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv10.c
@@ -52,10 +52,10 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv10_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv10_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
@@ -69,15 +69,15 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv10_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv10_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv10_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv10_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
@@ -88,15 +88,15 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv10_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv10_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv10_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv10_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
@@ -107,15 +107,15 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv1a_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv1a_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv10_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv10_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
@@ -126,15 +126,15 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv10_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv10_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv10_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv10_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
@@ -145,15 +145,15 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv10_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv10_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
@@ -164,15 +164,15 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv1a_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv1a_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
@@ -183,15 +183,15 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv10_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv10_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv20.c b/drivers/gpu/drm/nouveau/core/engine/device/nv20.c
index 12a4005fa619..dcde53b9f07f 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv20.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv20.c
@@ -53,15 +53,15 @@ nv20_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv20_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv20_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv20_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
@@ -72,15 +72,15 @@ nv20_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv25_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv25_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv25_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
@@ -91,15 +91,15 @@ nv20_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv25_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv25_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv25_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
@@ -110,15 +110,15 @@ nv20_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv25_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv25_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv2a_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv30.c b/drivers/gpu/drm/nouveau/core/engine/device/nv30.c
index cef0f1ea4c21..7b8662ef4f59 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv30.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv30.c
@@ -53,15 +53,15 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv30_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv30_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv30_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
@@ -72,15 +72,15 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv04_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv04_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv35_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv35_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv35_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
                 break;
@@ -91,15 +91,15 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv30_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv30_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv30_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv31_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
@@ -111,15 +111,15 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv20_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv36_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv36_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv35_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv31_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
@@ -131,15 +131,15 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv04_i2c_oclass;
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv04_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv10_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv04_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv10_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv10_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv04_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv17_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv34_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv31_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv40.c b/drivers/gpu/drm/nouveau/core/engine/device/nv40.c
index 1719cb0ee595..c8c41e93695e 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv40.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv40.c
@@ -35,6 +35,7 @@
 #include <subdev/fb.h>
 #include <subdev/instmem.h>
 #include <subdev/vm.h>
+#include <subdev/volt.h>
 
 #include <engine/device.h>
 #include <engine/dmaobj.h>
@@ -43,6 +44,7 @@
 #include <engine/graph.h>
 #include <engine/mpeg.h>
 #include <engine/disp.h>
+#include <engine/perfmon.h>
 
 int
 nv40_identify(struct nouveau_device *device)
@@ -56,18 +58,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv40_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv40_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x41:
                 device->cname = "NV41";
@@ -77,18 +81,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv40_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv41_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv41_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x42:
                 device->cname = "NV42";
@@ -98,18 +104,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv40_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv41_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv41_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x43:
                 device->cname = "NV43";
@@ -119,18 +127,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv40_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv41_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv41_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x45:
                 device->cname = "NV45";
@@ -140,18 +150,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv40_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv40_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv40_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
-                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x47:
                 device->cname = "G70";
@@ -161,18 +173,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv40_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv47_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv47_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
-                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x49:
                 device->cname = "G71";
@@ -182,18 +196,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv40_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv49_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv49_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
-                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x4b:
                 device->cname = "G73";
@@ -203,18 +219,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv04_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv40_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv49_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv49_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv41_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
-                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x44:
                 device->cname = "NV44";
@@ -224,18 +242,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv44_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv44_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv44_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
-                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x46:
                 device->cname = "G72";
@@ -245,18 +265,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv44_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv46_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv46_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
-                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x4a:
                 device->cname = "NV44A";
@@ -266,18 +288,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv44_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv44_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv44_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
-                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x4c:
                 device->cname = "C61";
@@ -287,18 +311,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv44_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv46_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv46_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
-                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x4e:
                 device->cname = "C51";
@@ -308,18 +334,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv44_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv4e_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv4e_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
-                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x63:
                 device->cname = "C73";
@@ -329,18 +357,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv44_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv46_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv46_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
-                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x67:
                 device->cname = "C67";
@@ -350,18 +380,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv44_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv46_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv46_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
-                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x68:
                 device->cname = "C68";
@@ -371,18 +403,20 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv40_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv40_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv1a_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv44_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv31_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv44_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv31_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv46_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv46_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv40_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv44_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv40_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv10_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv40_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
-                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
+                device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         default:
                 nv_fatal(device, "unknown Curie chipset\n");
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv50.c b/drivers/gpu/drm/nouveau/core/engine/device/nv50.c
index ffc18b80c5d9..db139827047c 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv50.c
@@ -36,6 +36,8 @@
 #include <subdev/instmem.h>
 #include <subdev/vm.h>
 #include <subdev/bar.h>
+#include <subdev/pwr.h>
+#include <subdev/volt.h>
 
 #include <engine/device.h>
 #include <engine/dmaobj.h>
@@ -49,6 +51,7 @@
 #include <engine/ppp.h>
 #include <engine/copy.h>
 #include <engine/disp.h>
+#include <engine/perfmon.h>
 
 int
 nv50_identify(struct nouveau_device *device)
@@ -59,257 +62,277 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
                 device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv50_i2c_oclass;
-                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] =  nv50_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv50_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv50_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv50_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv50_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv50_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv50_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv50_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv50_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv50_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv50_perfmon_oclass;
                 break;
         case 0x84:
                 device->cname = "G84";
                 device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
                 device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv50_i2c_oclass;
-                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] =  nv84_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv84_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv50_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv50_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv50_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv50_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv84_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv84_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv84_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0x86:
                 device->cname = "G86";
                 device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
                 device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv50_i2c_oclass;
-                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] =  nv84_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv84_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv50_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv50_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv50_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv50_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv84_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv84_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv84_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0x92:
                 device->cname = "G92";
                 device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
                 device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv50_i2c_oclass;
-                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] =  nv84_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv84_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv50_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv50_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv50_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv50_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv84_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv84_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv84_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0x94:
                 device->cname = "G94";
                 device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
                 device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv94_i2c_oclass;
-                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] =  nv84_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv84_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv50_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv50_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv94_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv94_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv84_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv84_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv94_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0x96:
                 device->cname = "G96";
                 device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
                 device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv94_i2c_oclass;
-                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] =  nv84_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv84_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv50_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv50_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv94_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv94_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv84_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv84_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv94_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0x98:
                 device->cname = "G98";
                 device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
                 device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv94_i2c_oclass;
-                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] =  nv84_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv84_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv50_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv98_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv94_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv84_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv98_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv98_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv98_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv94_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0xa0:
                 device->cname = "G200";
                 device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
                 device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv50_i2c_oclass;
-                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] =  nv84_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv84_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv50_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv98_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv94_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nv84_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv84_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nva0_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0xaa:
                 device->cname = "MCP77/MCP78";
                 device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
                 device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv94_i2c_oclass;
-                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] =  nv84_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv84_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv50_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv98_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv94_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nvaa_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv98_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv98_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv98_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv94_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0xac:
                 device->cname = "MCP79/MCP7A";
                 device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
                 device->oclass[NVDEV_SUBDEV_GPIO   ] = &nv50_gpio_oclass;
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nv94_i2c_oclass;
-                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nv50_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] =  nv84_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nv84_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nv50_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv50_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv98_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv94_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nvaa_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv98_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv98_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv98_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nv94_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0xa3:
                 device->cname = "GT215";
@@ -320,16 +343,18 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nva3_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nva3_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv50_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv98_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv94_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nva3_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nva3_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv84_mpeg_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv98_vp_oclass;
@@ -337,6 +362,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nva3_copy_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nva3_perfmon_oclass;
                 break;
         case 0xa5:
                 device->cname = "GT216";
@@ -347,22 +373,25 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nva3_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nva3_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv50_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv98_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv94_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nva3_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nva3_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv98_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv98_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nva3_copy_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nva3_perfmon_oclass;
                 break;
         case 0xa8:
                 device->cname = "GT218";
@@ -373,22 +402,25 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nva3_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nva3_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv50_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv98_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv94_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nva3_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nva3_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv98_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv98_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nva3_copy_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nva3_perfmon_oclass;
                 break;
         case 0xaf:
                 device->cname = "MCP89";
@@ -399,22 +431,25 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nva3_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nva3_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nv98_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nv50_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nv98_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nv94_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nv50_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nvaf_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv50_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nv50_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nva3_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv50_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nv84_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nv50_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv84_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv98_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv98_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nva3_copy_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] =  nva3_perfmon_oclass;
                 break;
         default:
                 nv_fatal(device, "unknown Tesla chipset\n");
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/device/nvc0.c
index 418f51f50d7a..606598f226fc 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nvc0.c
@@ -38,6 +38,8 @@
 #include <subdev/instmem.h>
 #include <subdev/vm.h>
 #include <subdev/bar.h>
+#include <subdev/pwr.h>
+#include <subdev/volt.h>
 
 #include <engine/device.h>
 #include <engine/dmaobj.h>
@@ -49,6 +51,7 @@
 #include <engine/ppp.h>
 #include <engine/copy.h>
 #include <engine/disp.h>
+#include <engine/perfmon.h>
 
 int
 nvc0_identify(struct nouveau_device *device)
@@ -63,18 +66,20 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nva3_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nvc0_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc0_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvc0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvc0_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nvc0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
@@ -82,6 +87,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xc4:
                 device->cname = "GF104";
@@ -92,18 +98,20 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nva3_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nvc0_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc0_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvc0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvc3_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nvc0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
@@ -111,6 +119,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xc3:
                 device->cname = "GF106";
@@ -121,24 +130,27 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nva3_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nvc0_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvc0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvc3_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nvc0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xce:
                 device->cname = "GF114";
@@ -149,18 +161,20 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nva3_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nvc0_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvc0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvc3_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nvc0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
@@ -168,6 +182,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xcf:
                 device->cname = "GF116";
@@ -178,18 +193,20 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nva3_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nvc0_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvc0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvc3_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nvc0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
@@ -197,6 +214,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xc1:
                 device->cname = "GF108";
@@ -207,24 +225,27 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nva3_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nvc0_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvc0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvc1_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nvc0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xc8:
                 device->cname = "GF110";
@@ -235,18 +256,20 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nva3_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nvc0_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvc0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvc8_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nvc0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
@@ -254,6 +277,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xd9:
                 device->cname = "GF119";
@@ -264,24 +288,27 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nvd0_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nvc0_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvd0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvd9_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nvc0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nvd0_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xd7:
                 device->cname = "GF117";
@@ -292,24 +319,25 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nvd0_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nvc0_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nvc0_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvd7_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nvc0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nvd0_disp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         default:
                 nv_fatal(device, "unknown Fermi chipset\n");
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nve0.c b/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
index 7aca1877add4..3900104976fc 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
@@ -38,6 +38,8 @@
 #include <subdev/instmem.h>
 #include <subdev/vm.h>
 #include <subdev/bar.h>
+#include <subdev/pwr.h>
+#include <subdev/volt.h>
 
 #include <engine/device.h>
 #include <engine/dmaobj.h>
@@ -49,6 +51,7 @@
 #include <engine/bsp.h>
 #include <engine/vp.h>
 #include <engine/ppp.h>
+#include <engine/perfmon.h>
 
 int
 nve0_identify(struct nouveau_device *device)
@@ -59,22 +62,24 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
                 device->oclass[NVDEV_SUBDEV_GPIO   ] = &nve0_gpio_oclass;
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nvd0_i2c_oclass;
-                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nvc0_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nve0_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nvd0_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nvc0_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvd0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nve0_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nve0_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nve4_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nve0_disp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nve0_copy0_oclass;
@@ -83,28 +88,31 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nve0_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nve0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] = &nve0_perfmon_oclass;
                 break;
         case 0xe7:
                 device->cname = "GK107";
                 device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
                 device->oclass[NVDEV_SUBDEV_GPIO   ] = &nve0_gpio_oclass;
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nvd0_i2c_oclass;
-                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nvc0_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nve0_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nvd0_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nvc0_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvd0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nve0_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nve0_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nve4_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nve0_disp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nve0_copy0_oclass;
@@ -113,28 +121,31 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nve0_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nve0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] = &nve0_perfmon_oclass;
                 break;
         case 0xe6:
                 device->cname = "GK106";
                 device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
                 device->oclass[NVDEV_SUBDEV_GPIO   ] = &nve0_gpio_oclass;
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nvd0_i2c_oclass;
-                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nvc0_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nve0_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nvd0_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nvc0_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvd0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nve0_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nve0_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nve4_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nve0_disp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nve0_copy0_oclass;
@@ -143,28 +154,31 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nve0_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nve0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
+                device->oclass[NVDEV_ENGINE_PERFMON] = &nve0_perfmon_oclass;
                 break;
         case 0xf0:
                 device->cname = "GK110";
                 device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
                 device->oclass[NVDEV_SUBDEV_GPIO   ] = &nve0_gpio_oclass;
                 device->oclass[NVDEV_SUBDEV_I2C    ] = &nvd0_i2c_oclass;
-                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nvc0_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nve0_clock_oclass;
                 device->oclass[NVDEV_SUBDEV_THERM  ] = &nvd0_therm_oclass;
                 device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
                 device->oclass[NVDEV_SUBDEV_DEVINIT] = &nvc0_devinit_oclass;
-                device->oclass[NVDEV_SUBDEV_MC     ] = &nvc0_mc_oclass;
-                device->oclass[NVDEV_SUBDEV_BUS    ] = &nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
-                device->oclass[NVDEV_SUBDEV_FB     ] = &nvc0_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
                 device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
                 device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nvd0_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
-                device->oclass[NVDEV_ENGINE_FIFO   ] = &nve0_fifo_oclass;
-                device->oclass[NVDEV_ENGINE_SW     ] = &nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nve0_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvf0_graph_oclass;
                 device->oclass[NVDEV_ENGINE_DISP   ] = &nvf0_disp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nve0_copy0_oclass;
@@ -175,6 +189,43 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_VP     ] = &nve0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
 #endif
+                device->oclass[NVDEV_ENGINE_PERFMON] = &nvf0_perfmon_oclass;
+                break;
+        case 0x108:
+                device->cname = "GK208";
+                device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+                device->oclass[NVDEV_SUBDEV_GPIO   ] = &nve0_gpio_oclass;
+                device->oclass[NVDEV_SUBDEV_I2C    ] = &nvd0_i2c_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nve0_clock_oclass;
+                device->oclass[NVDEV_SUBDEV_THERM  ] = &nvd0_therm_oclass;
+                device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+                device->oclass[NVDEV_SUBDEV_DEVINIT] = &nvc0_devinit_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
+                device->oclass[NVDEV_SUBDEV_INSTMEM] = &nv50_instmem_oclass;
+                device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nv108_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
+#if 0
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nve0_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_GR     ] =  nvf0_graph_oclass;
+#endif
+                device->oclass[NVDEV_ENGINE_DISP   ] = &nvf0_disp_oclass;
+#if 0
+                device->oclass[NVDEV_ENGINE_COPY0  ] = &nve0_copy0_oclass;
+                device->oclass[NVDEV_ENGINE_COPY1  ] = &nve0_copy1_oclass;
+                device->oclass[NVDEV_ENGINE_COPY2  ] = &nve0_copy2_oclass;
+                device->oclass[NVDEV_ENGINE_BSP    ] = &nve0_bsp_oclass;
+                device->oclass[NVDEV_ENGINE_VP     ] = &nve0_vp_oclass;
+                device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
+#endif
                 break;
         default:
                 nv_fatal(device, "unknown Kepler chipset\n");
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/priv.h b/drivers/gpu/drm/nouveau/core/engine/device/priv.h
new file mode 100644
index 000000000000..035fd5b9cfc3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/device/priv.h
@@ -0,0 +1,8 @@
+#ifndef __NVKM_DEVICE_PRIV_H__
+#define __NVKM_DEVICE_PRIV_H__
+
+#include <engine/device.h>
+
+extern struct nouveau_oclass nouveau_control_oclass[];
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/dport.c b/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
index 054d9cff4f53..1bd4c63369c1 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
@@ -70,17 +70,10 @@ dp_set_link_config(struct dp_state *dp)
         };
         u32 lnkcmp;
         u8 sink[2];
+        int ret;
 
         DBG("%d lanes at %d KB/s\n", dp->link_nr, dp->link_bw);
 
-        /* set desired link configuration on the sink */
-        sink[0] = dp->link_bw / 27000;
-        sink[1] = dp->link_nr;
-        if (dp->dpcd[DPCD_RC02] & DPCD_RC02_ENHANCED_FRAME_CAP)
-                sink[1] |= DPCD_LC01_ENHANCED_FRAME_EN;
-
-        nv_wraux(dp->aux, DPCD_LC00, sink, 2);
-
         /* set desired link configuration on the source */
         if ((lnkcmp = dp->info.lnkcmp)) {
                 if (dp->version < 0x30) {
@@ -96,10 +89,22 @@ dp_set_link_config(struct dp_state *dp)
                 nvbios_exec(&init);
         }
 
-        return dp->func->lnk_ctl(dp->disp, dp->outp, dp->head,
-                                 dp->link_nr, dp->link_bw / 27000,
-                                 dp->dpcd[DPCD_RC02] &
-                                          DPCD_RC02_ENHANCED_FRAME_CAP);
+        ret = dp->func->lnk_ctl(dp->disp, dp->outp, dp->head,
+                                dp->link_nr, dp->link_bw / 27000,
+                                dp->dpcd[DPCD_RC02] &
+                                         DPCD_RC02_ENHANCED_FRAME_CAP);
+        if (ret) {
+                ERR("lnk_ctl failed with %d\n", ret);
+                return ret;
+        }
+
+        /* set desired link configuration on the sink */
+        sink[0] = dp->link_bw / 27000;
+        sink[1] = dp->link_nr;
+        if (dp->dpcd[DPCD_RC02] & DPCD_RC02_ENHANCED_FRAME_CAP)
+                sink[1] |= DPCD_LC01_ENHANCED_FRAME_EN;
+
+        return nv_wraux(dp->aux, DPCD_LC00, sink, 2);
 }
 
 static void
@@ -294,8 +299,17 @@ nouveau_dp_train(struct nouveau_disp *disp, const struct nouveau_dp_func *func,
 
         ret = nv_rdaux(dp->aux, 0x00000, dp->dpcd, sizeof(dp->dpcd));
         if (ret) {
+                /* it's possible the display has been unplugged before we
+                 * get here.  we still need to execute the full set of
+                 * vbios scripts, and program the OR at a high enough
+                 * frequency to satisfy the target mode.  failure to do
+                 * so results at best in an UPDATE hanging, and at worst
+                 * with PDISP running away to join the circus.
+                 */
+                dp->dpcd[1] = link_bw[0] / 27000;
+                dp->dpcd[2] = 4;
+                dp->dpcd[3] = 0x00;
                 ERR("failed to read DPCD\n");
-                return ret;
         }
 
         /* adjust required bandwidth for 8B/10B coding overhead */
@@ -308,7 +322,7 @@ nouveau_dp_train(struct nouveau_disp *disp, const struct nouveau_dp_func *func,
         while (*link_bw > (dp->dpcd[1] * 27000))
                 link_bw++;
 
-        while (link_bw[0]) {
+        while ((ret = -EIO) && link_bw[0]) {
                 /* find minimum required lane count at this link rate */
                 dp->link_nr = dp->dpcd[2] & DPCD_RC02_MAX_LANE_COUNT;
                 while ((dp->link_nr >> 1) * link_bw[0] > datarate)
@@ -328,8 +342,10 @@ nouveau_dp_train(struct nouveau_disp *disp, const struct nouveau_dp_func *func,
                             !dp_link_train_eq(dp))
                                 break;
                 } else
-                if (ret >= 1) {
-                        /* dp_set_link_config() handled training */
+                if (ret) {
+                        /* dp_set_link_config() handled training, or
+                         * we failed to communicate with the sink.
+                         */
                         break;
                 }
 
@@ -339,8 +355,10 @@ nouveau_dp_train(struct nouveau_disp *disp, const struct nouveau_dp_func *func,
 
         /* finish link training */
         dp_set_training_pattern(dp, 0);
+        if (ret < 0)
+                ERR("link training failed\n");
 
         /* execute post-train script from vbios */
         dp_link_train_fini(dp);
-        return true;
+        return (ret < 0) ? false : true;
 }
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c
index 05e903f08a36..a0bc8a89b699 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c
@@ -59,6 +59,7 @@ nv04_disp_intr(struct nouveau_subdev *subdev)
         struct nv04_disp_priv *priv = (void *)subdev;
         u32 crtc0 = nv_rd32(priv, 0x600100);
         u32 crtc1 = nv_rd32(priv, 0x602100);
+        u32 pvideo;
 
         if (crtc0 & 0x00000001) {
                 nouveau_event_trigger(priv->base.vblank, 0);
@@ -69,6 +70,14 @@ nv04_disp_intr(struct nouveau_subdev *subdev)
                 nouveau_event_trigger(priv->base.vblank, 1);
                 nv_wr32(priv, 0x602100, 0x00000001);
         }
+
+        if (nv_device(priv)->chipset >= 0x10 &&
+            nv_device(priv)->chipset <= 0x40) {
+                pvideo = nv_rd32(priv, 0x8100);
+                if (pvideo & ~0x11)
+                        nv_info(priv, "PVIDEO intr: %08x\n", pvideo);
+                nv_wr32(priv, 0x8100, pvideo);
+        }
 }
 
 static int
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
index 52dd7a1db729..378a015091d2 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
@@ -541,6 +541,15 @@ nvd0_disp_base_init(struct nouveau_object *object)
         nv_wr32(priv, 0x6100a0, 0x00000000);
         nv_wr32(priv, 0x6100b0, 0x00000307);
 
+        /* disable underflow reporting, preventing an intermittent issue
+         * on some nve4 boards where the production vbios left this
+         * setting enabled by default.
+         *
+         * ftp://download.nvidia.com/open-gpu-doc/gk104-disable-underflow-reporting/1/gk104-disable-underflow-reporting.txt
+         */
+        for (i = 0; i < priv->head.nr; i++)
+                nv_mask(priv, 0x616308 + (i * 0x800), 0x00000111, 0x00000010);
+
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/sornv94.c b/drivers/gpu/drm/nouveau/core/engine/disp/sornv94.c
index 7ec4ee83fb64..eea3ef59693d 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/sornv94.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/sornv94.c
@@ -97,8 +97,9 @@ nv94_sor_dp_drv_ctl(struct nouveau_disp *disp, struct dcb_output *outp,
 {
         struct nouveau_bios *bios = nouveau_bios(disp);
         struct nv50_disp_priv *priv = (void *)disp;
+        const u32 shift = nv94_sor_dp_lane_map(priv, lane);
         const u32 loff = nv94_sor_loff(outp);
-        u32 addr, shift = nv94_sor_dp_lane_map(priv, lane);
+        u32 addr, data[3];
         u8  ver, hdr, cnt, len;
         struct nvbios_dpout info;
         struct nvbios_dpcfg ocfg;
@@ -113,9 +114,12 @@ nv94_sor_dp_drv_ctl(struct nouveau_disp *disp, struct dcb_output *outp,
         if (!addr)
                 return -EINVAL;
 
-        nv_mask(priv, 0x61c118 + loff, 0x000000ff << shift, ocfg.drv << shift);
-        nv_mask(priv, 0x61c120 + loff, 0x000000ff << shift, ocfg.pre << shift);
-        nv_mask(priv, 0x61c130 + loff, 0x0000ff00, ocfg.unk << 8);
+        data[0] = nv_rd32(priv, 0x61c118 + loff) & ~(0x000000ff << shift);
+        data[1] = nv_rd32(priv, 0x61c120 + loff) & ~(0x000000ff << shift);
+        data[2] = nv_rd32(priv, 0x61c130 + loff) & ~(0x0000ff00);
+        nv_wr32(priv, 0x61c118 + loff, data[0] | (ocfg.drv << shift));
+        nv_wr32(priv, 0x61c120 + loff, data[1] | (ocfg.pre << shift));
+        nv_wr32(priv, 0x61c130 + loff, data[2] | (ocfg.unk << 8));
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c b/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c
index 9e1d435d7282..d2df572f16a3 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/sornvd0.c
@@ -93,8 +93,9 @@ nvd0_sor_dp_drv_ctl(struct nouveau_disp *disp, struct dcb_output *outp,
 {
         struct nouveau_bios *bios = nouveau_bios(disp);
         struct nv50_disp_priv *priv = (void *)disp;
+        const u32 shift = nvd0_sor_dp_lane_map(priv, lane);
         const u32 loff = nvd0_sor_loff(outp);
-        u32 addr, shift = nvd0_sor_dp_lane_map(priv, lane);
+        u32 addr, data[3];
         u8  ver, hdr, cnt, len;
         struct nvbios_dpout info;
         struct nvbios_dpcfg ocfg;
@@ -109,9 +110,12 @@ nvd0_sor_dp_drv_ctl(struct nouveau_disp *disp, struct dcb_output *outp,
         if (!addr)
                 return -EINVAL;
 
-        nv_mask(priv, 0x61c118 + loff, 0x000000ff << shift, ocfg.drv << shift);
-        nv_mask(priv, 0x61c120 + loff, 0x000000ff << shift, ocfg.pre << shift);
-        nv_mask(priv, 0x61c130 + loff, 0x0000ff00, ocfg.unk << 8);
+        data[0] = nv_rd32(priv, 0x61c118 + loff) & ~(0x000000ff << shift);
+        data[1] = nv_rd32(priv, 0x61c120 + loff) & ~(0x000000ff << shift);
+        data[2] = nv_rd32(priv, 0x61c130 + loff) & ~(0x0000ff00);
+        nv_wr32(priv, 0x61c118 + loff, data[0] | (ocfg.drv << shift));
+        nv_wr32(priv, 0x61c120 + loff, data[1] | (ocfg.pre << shift));
+        nv_wr32(priv, 0x61c130 + loff, data[2] | (ocfg.unk << 8));
         nv_mask(priv, 0x61c13c + loff, 0x00000000, 0x00000000);
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c
index f877bd524a92..54f26cc801c7 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv04.c
@@ -632,8 +632,8 @@ nv04_fifo_init(struct nouveau_object *object)
         return 0;
 }
 
-struct nouveau_oclass
-nv04_fifo_oclass = {
+struct nouveau_oclass *
+nv04_fifo_oclass = &(struct nouveau_oclass) {
         .handle = NV_ENGINE(FIFO, 0x04),
         .ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv04_fifo_ctor,
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c
index 2c927c1d173b..571a22aa1ae5 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv10.c
@@ -159,8 +159,8 @@ nv10_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-struct nouveau_oclass
-nv10_fifo_oclass = {
+struct nouveau_oclass *
+nv10_fifo_oclass = &(struct nouveau_oclass) {
         .handle = NV_ENGINE(FIFO, 0x10),
         .ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv10_fifo_ctor,
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv17.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv17.c
index a9cb51d38c57..f25760209316 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nv17.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv17.c
@@ -196,8 +196,8 @@ nv17_fifo_init(struct nouveau_object *object)
         return 0;
 }
 
-struct nouveau_oclass
-nv17_fifo_oclass = {
+struct nouveau_oclass *
+nv17_fifo_oclass = &(struct nouveau_oclass) {
         .handle = NV_ENGINE(FIFO, 0x17),
         .ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv17_fifo_ctor,
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv40.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv40.c
index 5c7433d5069f..343487ed2238 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nv40.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv40.c
@@ -337,8 +337,8 @@ nv40_fifo_init(struct nouveau_object *object)
         return 0;
 }
 
-struct nouveau_oclass
-nv40_fifo_oclass = {
+struct nouveau_oclass *
+nv40_fifo_oclass = &(struct nouveau_oclass) {
         .handle = NV_ENGINE(FIFO, 0x40),
         .ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv40_fifo_ctor,
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv50.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv50.c
index 7e5dff51d3c5..5f555788121c 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv50.c
@@ -502,8 +502,8 @@ nv50_fifo_init(struct nouveau_object *object)
         return 0;
 }
 
-struct nouveau_oclass
-nv50_fifo_oclass = {
+struct nouveau_oclass *
+nv50_fifo_oclass = &(struct nouveau_oclass) {
         .handle = NV_ENGINE(FIFO, 0x50),
         .ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv50_fifo_ctor,
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nv84.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nv84.c
index 91a87cd7195a..0908dc834c84 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nv84.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nv84.c
@@ -144,7 +144,7 @@ nv84_fifo_object_attach(struct nouveau_object *parent,
         case NVDEV_ENGINE_COPY0 : context |= 0x00300000; break;
         case NVDEV_ENGINE_VP    : context |= 0x00400000; break;
         case NVDEV_ENGINE_CRYPT :
-        case NVDEV_ENGINE_UNK1C1: context |= 0x00500000; break;
+        case NVDEV_ENGINE_VIC   : context |= 0x00500000; break;
         case NVDEV_ENGINE_BSP   : context |= 0x00600000; break;
         default:
                 return -EINVAL;
@@ -180,7 +180,7 @@ nv84_fifo_chan_ctor_dma(struct nouveau_object *parent,
                                           (1ULL << NVDEV_ENGINE_BSP) |
                                           (1ULL << NVDEV_ENGINE_PPP) |
                                           (1ULL << NVDEV_ENGINE_COPY0) |
-                                          (1ULL << NVDEV_ENGINE_UNK1C1), &chan);
+                                          (1ULL << NVDEV_ENGINE_VIC), &chan);
         *pobject = nv_object(chan);
         if (ret)
                 return ret;
@@ -243,7 +243,7 @@ nv84_fifo_chan_ctor_ind(struct nouveau_object *parent,
                                           (1ULL << NVDEV_ENGINE_BSP) |
                                           (1ULL << NVDEV_ENGINE_PPP) |
                                           (1ULL << NVDEV_ENGINE_COPY0) |
-                                          (1ULL << NVDEV_ENGINE_UNK1C1), &chan);
+                                          (1ULL << NVDEV_ENGINE_VIC), &chan);
         *pobject = nv_object(chan);
         if (ret)
                 return ret;
@@ -435,8 +435,8 @@ nv84_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-struct nouveau_oclass
-nv84_fifo_oclass = {
+struct nouveau_oclass *
+nv84_fifo_oclass = &(struct nouveau_oclass) {
         .handle = NV_ENGINE(FIFO, 0x84),
         .ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv84_fifo_ctor,
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c
index ce92f289e751..e21453a94971 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c
@@ -720,8 +720,8 @@ nvc0_fifo_init(struct nouveau_object *object)
         return 0;
 }
 
-struct nouveau_oclass
-nvc0_fifo_oclass = {
+struct nouveau_oclass *
+nvc0_fifo_oclass = &(struct nouveau_oclass) {
         .handle = NV_ENGINE(FIFO, 0xc0),
         .ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nvc0_fifo_ctor,
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
index 8e8121abe31b..fcd449e5aba7 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
@@ -675,8 +675,8 @@ nve0_fifo_init(struct nouveau_object *object)
         return 0;
 }
 
-struct nouveau_oclass
-nve0_fifo_oclass = {
+struct nouveau_oclass *
+nve0_fifo_oclass = &(struct nouveau_oclass) {
         .handle = NV_ENGINE(FIFO, 0xe0),
         .ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nve0_fifo_ctor,
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc1.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc1.c
index e5be3ee7f172..71b4283f7fad 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc1.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc1.c
@@ -587,6 +587,7 @@ nvc1_grctx_init_unk58xx[] = {
         { 0x405870,   4, 0x04, 0x00000001 },
         { 0x405a00,   2, 0x04, 0x00000000 },
         { 0x405a18,   1, 0x04, 0x00000000 },
+        {}
 };
 
 static struct nvc0_graph_init
@@ -598,6 +599,7 @@ nvc1_grctx_init_rop[] = {
         { 0x408904,   1, 0x04, 0x62000001 },
         { 0x408908,   1, 0x04, 0x00c80929 },
         { 0x408980,   1, 0x04, 0x0000011d },
+        {}
 };
 
 static struct nvc0_graph_init
@@ -671,6 +673,7 @@ nvc1_grctx_init_gpc_0[] = {
         { 0x419000,   1, 0x04, 0x00000780 },
         { 0x419004,   2, 0x04, 0x00000000 },
         { 0x419014,   1, 0x04, 0x00000004 },
+        {}
 };
 
 static struct nvc0_graph_init
@@ -717,6 +720,7 @@ nvc1_grctx_init_tpc[] = {
         { 0x419e98,   1, 0x04, 0x00000000 },
         { 0x419ee0,   1, 0x04, 0x00011110 },
         { 0x419f30,  11, 0x04, 0x00000000 },
+        {}
 };
 
 void
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd7.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd7.c
index 438e78410808..c4740d528532 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd7.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd7.c
@@ -258,6 +258,7 @@ nvd7_grctx_init_hub[] = {
         nvc0_grctx_init_unk78xx,
         nvc0_grctx_init_unk80xx,
         nvd9_grctx_init_rop,
+        NULL
 };
 
 struct nvc0_graph_init *
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd9.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd9.c
index 818a4751df46..a1102cbf2fdc 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd9.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd9.c
@@ -466,6 +466,7 @@ nvd9_grctx_init_hub[] = {
         nvc0_grctx_init_unk78xx,
         nvc0_grctx_init_unk80xx,
         nvd9_grctx_init_rop,
+        NULL
 };
 
 struct nvc0_graph_init *
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv10.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv10.c
index 23c143aaa556..4532f7e5618c 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv10.c
@@ -945,7 +945,8 @@ nv10_graph_load_context(struct nv10_graph_chan *chan, int chid)
         for (i = 0; i < ARRAY_SIZE(nv10_graph_ctx_regs); i++)
                 nv_wr32(priv, nv10_graph_ctx_regs[i], chan->nv10[i]);
 
-        if (nv_device(priv)->chipset >= 0x17) {
+        if (nv_device(priv)->card_type >= NV_11 &&
+            nv_device(priv)->chipset >= 0x17) {
                 for (i = 0; i < ARRAY_SIZE(nv17_graph_ctx_regs); i++)
                         nv_wr32(priv, nv17_graph_ctx_regs[i], chan->nv17[i]);
         }
@@ -970,7 +971,8 @@ nv10_graph_unload_context(struct nv10_graph_chan *chan)
         for (i = 0; i < ARRAY_SIZE(nv10_graph_ctx_regs); i++)
                 chan->nv10[i] = nv_rd32(priv, nv10_graph_ctx_regs[i]);
 
-        if (nv_device(priv)->chipset >= 0x17) {
+        if (nv_device(priv)->card_type >= NV_11 &&
+            nv_device(priv)->chipset >= 0x17) {
                 for (i = 0; i < ARRAY_SIZE(nv17_graph_ctx_regs); i++)
                         chan->nv17[i] = nv_rd32(priv, nv17_graph_ctx_regs[i]);
         }
@@ -1052,7 +1054,8 @@ nv10_graph_context_ctor(struct nouveau_object *parent,
         NV_WRITE_CTX(0x00400e14, 0x00001000);
         NV_WRITE_CTX(0x00400e30, 0x00080008);
         NV_WRITE_CTX(0x00400e34, 0x00080008);
-        if (nv_device(priv)->chipset >= 0x17) {
+        if (nv_device(priv)->card_type >= NV_11 &&
+            nv_device(priv)->chipset >= 0x17) {
                 /* is it really needed ??? */
                 NV17_WRITE_CTX(NV10_PGRAPH_DEBUG_4,
                                         nv_rd32(priv, NV10_PGRAPH_DEBUG_4));
@@ -1231,7 +1234,7 @@ nv10_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                 nv_engine(priv)->sclass = nv10_graph_sclass;
         else
         if (nv_device(priv)->chipset <  0x17 ||
-            nv_device(priv)->chipset == 0x1a)
+            nv_device(priv)->card_type < NV_11)
                 nv_engine(priv)->sclass = nv15_graph_sclass;
         else
                 nv_engine(priv)->sclass = nv17_graph_sclass;
@@ -1270,7 +1273,8 @@ nv10_graph_init(struct nouveau_object *object)
         nv_wr32(priv, NV04_PGRAPH_DEBUG_2, 0x25f92ad9);
         nv_wr32(priv, NV04_PGRAPH_DEBUG_3, 0x55DE0830 | (1 << 29) | (1 << 31));
 
-        if (nv_device(priv)->chipset >= 0x17) {
+        if (nv_device(priv)->card_type >= NV_11 &&
+            nv_device(priv)->chipset >= 0x17) {
                 nv_wr32(priv, NV10_PGRAPH_DEBUG_4, 0x1f000000);
                 nv_wr32(priv, 0x400a10, 0x03ff3fb6);
                 nv_wr32(priv, 0x400838, 0x002f8684);
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
index 3f4f35cc3848..434bb4b0fa2e 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
@@ -1138,7 +1138,7 @@ nvc0_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        nv_subdev(priv)->unit = 0x18001000;
+        nv_subdev(priv)->unit = 0x08001000;
         nv_subdev(priv)->intr = nvc0_graph_intr;
 
         priv->base.units = nvc0_graph_units;
diff --git a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv31.c b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv31.c
index c19004301309..7eb6d94c84e2 100644
--- a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv31.c
+++ b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv31.c
@@ -34,16 +34,7 @@
 
 #include <engine/fifo.h>
 #include <engine/mpeg.h>
-#include <engine/graph/nv40.h>
-
-struct nv31_mpeg_priv {
-        struct nouveau_mpeg base;
-        atomic_t refcount;
-};
-
-struct nv31_mpeg_chan {
-        struct nouveau_object base;
-};
+#include <engine/mpeg/nv31.h>
 
 /*******************************************************************************
  * MPEG object classes
@@ -89,18 +80,18 @@ nv31_mpeg_mthd_dma(struct nouveau_object *object, u32 mthd, void *arg, u32 len)
 
         if (mthd == 0x0190) {
                 /* DMA_CMD */
-                nv_mask(priv, 0x00b300, 0x00030000, (dma0 & 0x00030000));
+                nv_mask(priv, 0x00b300, 0x00010000, (dma0 & 0x00030000) ? 0x00010000 : 0);
                 nv_wr32(priv, 0x00b334, base);
                 nv_wr32(priv, 0x00b324, size);
         } else
         if (mthd == 0x01a0) {
                 /* DMA_DATA */
-                nv_mask(priv, 0x00b300, 0x000c0000, (dma0 & 0x00030000) << 2);
+                nv_mask(priv, 0x00b300, 0x00020000, (dma0 & 0x00030000) ? 0x00020000 : 0);
                 nv_wr32(priv, 0x00b360, base);
                 nv_wr32(priv, 0x00b364, size);
         } else {
                 /* DMA_IMAGE, VRAM only */
-                if (dma0 & 0x000c0000)
+                if (dma0 & 0x00030000)
                         return -EINVAL;
 
                 nv_wr32(priv, 0x00b370, base);
@@ -110,7 +101,7 @@ nv31_mpeg_mthd_dma(struct nouveau_object *object, u32 mthd, void *arg, u32 len)
         return 0;
 }
 
-static struct nouveau_ofuncs
+struct nouveau_ofuncs
 nv31_mpeg_ofuncs = {
         .ctor = nv31_mpeg_object_ctor,
         .dtor = _nouveau_gpuobj_dtor,
@@ -146,16 +137,23 @@ nv31_mpeg_context_ctor(struct nouveau_object *parent,
 {
         struct nv31_mpeg_priv *priv = (void *)engine;
         struct nv31_mpeg_chan *chan;
+        unsigned long flags;
         int ret;
 
-        if (!atomic_add_unless(&priv->refcount, 1, 1))
-                return -EBUSY;
-
         ret = nouveau_object_create(parent, engine, oclass, 0, &chan);
         *pobject = nv_object(chan);
         if (ret)
                 return ret;
 
+        spin_lock_irqsave(&nv_engine(priv)->lock, flags);
+        if (priv->chan) {
+                spin_unlock_irqrestore(&nv_engine(priv)->lock, flags);
+                nouveau_object_destroy(&chan->base);
+                *pobject = NULL;
+                return -EBUSY;
+        }
+        priv->chan = chan;
+        spin_unlock_irqrestore(&nv_engine(priv)->lock, flags);
         return 0;
 }
 
@@ -164,11 +162,15 @@ nv31_mpeg_context_dtor(struct nouveau_object *object)
 {
         struct nv31_mpeg_priv *priv = (void *)object->engine;
         struct nv31_mpeg_chan *chan = (void *)object;
-        atomic_dec(&priv->refcount);
+        unsigned long flags;
+
+        spin_lock_irqsave(&nv_engine(priv)->lock, flags);
+        priv->chan = NULL;
+        spin_unlock_irqrestore(&nv_engine(priv)->lock, flags);
         nouveau_object_destroy(&chan->base);
 }
 
-static struct nouveau_oclass
+struct nouveau_oclass
 nv31_mpeg_cclass = {
         .handle = NV_ENGCTX(MPEG, 0x31),
         .ofuncs = &(struct nouveau_ofuncs) {
@@ -197,21 +199,19 @@ nv31_mpeg_tile_prog(struct nouveau_engine *engine, int i)
 void
 nv31_mpeg_intr(struct nouveau_subdev *subdev)
 {
+        struct nv31_mpeg_priv *priv = (void *)subdev;
         struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
-        struct nouveau_engine *engine = nv_engine(subdev);
-        struct nouveau_object *engctx;
         struct nouveau_handle *handle;
-        struct nv31_mpeg_priv *priv = (void *)subdev;
-        u32 inst = nv_rd32(priv, 0x00b318) & 0x000fffff;
+        struct nouveau_object *engctx;
         u32 stat = nv_rd32(priv, 0x00b100);
         u32 type = nv_rd32(priv, 0x00b230);
         u32 mthd = nv_rd32(priv, 0x00b234);
         u32 data = nv_rd32(priv, 0x00b238);
         u32 show = stat;
-        int chid;
+        unsigned long flags;
 
-        engctx = nouveau_engctx_get(engine, inst);
-        chid   = pfifo->chid(pfifo, engctx);
+        spin_lock_irqsave(&nv_engine(priv)->lock, flags);
+        engctx = nv_object(priv->chan);
 
         if (stat & 0x01000000) {
                 /* happens on initial binding of the object */
@@ -220,7 +220,7 @@ nv31_mpeg_intr(struct nouveau_subdev *subdev)
                         show &= ~0x01000000;
                 }
 
-                if (type == 0x00000010) {
+                if (type == 0x00000010 && engctx) {
                         handle = nouveau_handle_get_class(engctx, 0x3174);
                         if (handle && !nv_call(handle->object, mthd, data))
                                 show &= ~0x01000000;
@@ -232,13 +232,12 @@ nv31_mpeg_intr(struct nouveau_subdev *subdev)
         nv_wr32(priv, 0x00b230, 0x00000001);
 
         if (show) {
-                nv_error(priv,
-                         "ch %d [0x%08x %s] 0x%08x 0x%08x 0x%08x 0x%08x\n",
-                         chid, inst << 4, nouveau_client_name(engctx), stat,
-                         type, mthd, data);
+                nv_error(priv, "ch %d [%s] 0x%08x 0x%08x 0x%08x 0x%08x\n",
+                         pfifo->chid(pfifo, engctx),
+                         nouveau_client_name(engctx), stat, type, mthd, data);
         }
 
-        nouveau_engctx_put(engctx);
+        spin_unlock_irqrestore(&nv_engine(priv)->lock, flags);
 }
 
 static int
@@ -284,10 +283,7 @@ nv31_mpeg_init(struct nouveau_object *object)
         /* PMPEG init */
         nv_wr32(priv, 0x00b32c, 0x00000000);
         nv_wr32(priv, 0x00b314, 0x00000100);
-        if (nv_device(priv)->chipset >= 0x40 && nv44_graph_class(priv))
-                nv_wr32(priv, 0x00b220, 0x00000044);
-        else
-                nv_wr32(priv, 0x00b220, 0x00000031);
+        nv_wr32(priv, 0x00b220, 0x00000031);
         nv_wr32(priv, 0x00b300, 0x02001ec1);
         nv_mask(priv, 0x00b32c, 0x00000001, 0x00000001);
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv31.h b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv31.h
new file mode 100644
index 000000000000..d08629d0b6ad
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv31.h
@@ -0,0 +1,15 @@
+#ifndef __NV31_MPEG_H__
+#define __NV31_MPEG_H__
+
+#include <engine/mpeg.h>
+
+struct nv31_mpeg_chan {
+        struct nouveau_object base;
+};
+
+struct nv31_mpeg_priv {
+        struct nouveau_mpeg base;
+        struct nv31_mpeg_chan *chan;
+};
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv40.c b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv40.c
index dd6196072e9c..d4e7ec0ba68c 100644
--- a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv40.c
+++ b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv40.c
@@ -31,66 +31,63 @@
 #include <subdev/instmem.h>
 
 #include <engine/mpeg.h>
-#include <engine/graph/nv40.h>
-
-struct nv40_mpeg_priv {
-        struct nouveau_mpeg base;
-};
-
-struct nv40_mpeg_chan {
-        struct nouveau_mpeg_chan base;
-};
+#include <engine/mpeg/nv31.h>
 
 /*******************************************************************************
- * PMPEG context
+ * MPEG object classes
  ******************************************************************************/
 
 static int
-nv40_mpeg_context_ctor(struct nouveau_object *parent,
-                       struct nouveau_object *engine,
-                       struct nouveau_oclass *oclass, void *data, u32 size,
-                       struct nouveau_object **pobject)
+nv40_mpeg_mthd_dma(struct nouveau_object *object, u32 mthd, void *arg, u32 len)
 {
-        struct nv40_mpeg_chan *chan;
-        int ret;
-
-        ret = nouveau_mpeg_context_create(parent, engine, oclass, NULL,
-                                          264 * 4, 16,
-                                          NVOBJ_FLAG_ZERO_ALLOC, &chan);
-        *pobject = nv_object(chan);
-        if (ret)
-                return ret;
+        struct nouveau_instmem *imem = nouveau_instmem(object);
+        struct nv31_mpeg_priv *priv = (void *)object->engine;
+        u32 inst = *(u32 *)arg << 4;
+        u32 dma0 = nv_ro32(imem, inst + 0);
+        u32 dma1 = nv_ro32(imem, inst + 4);
+        u32 dma2 = nv_ro32(imem, inst + 8);
+        u32 base = (dma2 & 0xfffff000) | (dma0 >> 20);
+        u32 size = dma1 + 1;
+
+        /* only allow linear DMA objects */
+        if (!(dma0 & 0x00002000))
+                return -EINVAL;
+
+        if (mthd == 0x0190) {
+                /* DMA_CMD */
+                nv_mask(priv, 0x00b300, 0x00030000, (dma0 & 0x00030000));
+                nv_wr32(priv, 0x00b334, base);
+                nv_wr32(priv, 0x00b324, size);
+        } else
+        if (mthd == 0x01a0) {
+                /* DMA_DATA */
+                nv_mask(priv, 0x00b300, 0x000c0000, (dma0 & 0x00030000) << 2);
+                nv_wr32(priv, 0x00b360, base);
+                nv_wr32(priv, 0x00b364, size);
+        } else {
+                /* DMA_IMAGE, VRAM only */
+                if (dma0 & 0x00030000)
+                        return -EINVAL;
+
+                nv_wr32(priv, 0x00b370, base);
+                nv_wr32(priv, 0x00b374, size);
+        }
 
-        nv_wo32(&chan->base.base, 0x78, 0x02001ec1);
         return 0;
 }
 
-static int
-nv40_mpeg_context_fini(struct nouveau_object *object, bool suspend)
-{
-
-        struct nv40_mpeg_priv *priv = (void *)object->engine;
-        struct nv40_mpeg_chan *chan = (void *)object;
-        u32 inst = 0x80000000 | nv_gpuobj(chan)->addr >> 4;
-
-        nv_mask(priv, 0x00b32c, 0x00000001, 0x00000000);
-        if (nv_rd32(priv, 0x00b318) == inst)
-                nv_mask(priv, 0x00b318, 0x80000000, 0x00000000);
-        nv_mask(priv, 0x00b32c, 0x00000001, 0x00000001);
-        return 0;
-}
+static struct nouveau_omthds
+nv40_mpeg_omthds[] = {
+        { 0x0190, 0x0190, nv40_mpeg_mthd_dma },
+        { 0x01a0, 0x01a0, nv40_mpeg_mthd_dma },
+        { 0x01b0, 0x01b0, nv40_mpeg_mthd_dma },
+        {}
+};
 
-static struct nouveau_oclass
-nv40_mpeg_cclass = {
-        .handle = NV_ENGCTX(MPEG, 0x40),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv40_mpeg_context_ctor,
-                .dtor = _nouveau_mpeg_context_dtor,
-                .init = _nouveau_mpeg_context_init,
-                .fini = nv40_mpeg_context_fini,
-                .rd32 = _nouveau_mpeg_context_rd32,
-                .wr32 = _nouveau_mpeg_context_wr32,
-        },
+struct nouveau_oclass
+nv40_mpeg_sclass[] = {
+        { 0x3174, &nv31_mpeg_ofuncs, nv40_mpeg_omthds },
+        {}
 };
 
 /*******************************************************************************
@@ -100,7 +97,7 @@ nv40_mpeg_cclass = {
 static void
 nv40_mpeg_intr(struct nouveau_subdev *subdev)
 {
-        struct nv40_mpeg_priv *priv = (void *)subdev;
+        struct nv31_mpeg_priv *priv = (void *)subdev;
         u32 stat;
 
         if ((stat = nv_rd32(priv, 0x00b100)))
@@ -117,7 +114,7 @@ nv40_mpeg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                struct nouveau_oclass *oclass, void *data, u32 size,
                struct nouveau_object **pobject)
 {
-        struct nv40_mpeg_priv *priv;
+        struct nv31_mpeg_priv *priv;
         int ret;
 
         ret = nouveau_mpeg_create(parent, engine, oclass, &priv);
@@ -127,8 +124,8 @@ nv40_mpeg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 
         nv_subdev(priv)->unit = 0x00000002;
         nv_subdev(priv)->intr = nv40_mpeg_intr;
-        nv_engine(priv)->cclass = &nv40_mpeg_cclass;
-        nv_engine(priv)->sclass = nv31_mpeg_sclass;
+        nv_engine(priv)->cclass = &nv31_mpeg_cclass;
+        nv_engine(priv)->sclass = nv40_mpeg_sclass;
         nv_engine(priv)->tile_prog = nv31_mpeg_tile_prog;
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/core/engine/mpeg/nv44.c b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv44.c
new file mode 100644
index 000000000000..3d8c2133e0e8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/mpeg/nv44.c
@@ -0,0 +1,194 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/os.h>
+#include <core/class.h>
+#include <core/client.h>
+#include <core/engctx.h>
+#include <core/handle.h>
+
+#include <subdev/fb.h>
+#include <subdev/timer.h>
+#include <subdev/instmem.h>
+
+#include <engine/fifo.h>
+#include <engine/mpeg.h>
+
+struct nv44_mpeg_priv {
+        struct nouveau_mpeg base;
+};
+
+struct nv44_mpeg_chan {
+        struct nouveau_mpeg_chan base;
+};
+
+/*******************************************************************************
+ * PMPEG context
+ ******************************************************************************/
+
+static int
+nv44_mpeg_context_ctor(struct nouveau_object *parent,
+                       struct nouveau_object *engine,
+                       struct nouveau_oclass *oclass, void *data, u32 size,
+                       struct nouveau_object **pobject)
+{
+        struct nv44_mpeg_chan *chan;
+        int ret;
+
+        ret = nouveau_mpeg_context_create(parent, engine, oclass, NULL,
+                                          264 * 4, 16,
+                                          NVOBJ_FLAG_ZERO_ALLOC, &chan);
+        *pobject = nv_object(chan);
+        if (ret)
+                return ret;
+
+        nv_wo32(&chan->base.base, 0x78, 0x02001ec1);
+        return 0;
+}
+
+static int
+nv44_mpeg_context_fini(struct nouveau_object *object, bool suspend)
+{
+
+        struct nv44_mpeg_priv *priv = (void *)object->engine;
+        struct nv44_mpeg_chan *chan = (void *)object;
+        u32 inst = 0x80000000 | nv_gpuobj(chan)->addr >> 4;
+
+        nv_mask(priv, 0x00b32c, 0x00000001, 0x00000000);
+        if (nv_rd32(priv, 0x00b318) == inst)
+                nv_mask(priv, 0x00b318, 0x80000000, 0x00000000);
+        nv_mask(priv, 0x00b32c, 0x00000001, 0x00000001);
+        return 0;
+}
+
+static struct nouveau_oclass
+nv44_mpeg_cclass = {
+        .handle = NV_ENGCTX(MPEG, 0x44),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv44_mpeg_context_ctor,
+                .dtor = _nouveau_mpeg_context_dtor,
+                .init = _nouveau_mpeg_context_init,
+                .fini = nv44_mpeg_context_fini,
+                .rd32 = _nouveau_mpeg_context_rd32,
+                .wr32 = _nouveau_mpeg_context_wr32,
+        },
+};
+
+/*******************************************************************************
+ * PMPEG engine/subdev functions
+ ******************************************************************************/
+
+static void
+nv44_mpeg_intr(struct nouveau_subdev *subdev)
+{
+        struct nouveau_fifo *pfifo = nouveau_fifo(subdev);
+        struct nouveau_engine *engine = nv_engine(subdev);
+        struct nouveau_object *engctx;
+        struct nouveau_handle *handle;
+        struct nv44_mpeg_priv *priv = (void *)subdev;
+        u32 inst = nv_rd32(priv, 0x00b318) & 0x000fffff;
+        u32 stat = nv_rd32(priv, 0x00b100);
+        u32 type = nv_rd32(priv, 0x00b230);
+        u32 mthd = nv_rd32(priv, 0x00b234);
+        u32 data = nv_rd32(priv, 0x00b238);
+        u32 show = stat;
+        int chid;
+
+        engctx = nouveau_engctx_get(engine, inst);
+        chid   = pfifo->chid(pfifo, engctx);
+
+        if (stat & 0x01000000) {
+                /* happens on initial binding of the object */
+                if (type == 0x00000020 && mthd == 0x0000) {
+                        nv_mask(priv, 0x00b308, 0x00000000, 0x00000000);
+                        show &= ~0x01000000;
+                }
+
+                if (type == 0x00000010) {
+                        handle = nouveau_handle_get_class(engctx, 0x3174);
+                        if (handle && !nv_call(handle->object, mthd, data))
+                                show &= ~0x01000000;
+                        nouveau_handle_put(handle);
+                }
+        }
+
+        nv_wr32(priv, 0x00b100, stat);
+        nv_wr32(priv, 0x00b230, 0x00000001);
+
+        if (show) {
+                nv_error(priv,
+                         "ch %d [0x%08x %s] 0x%08x 0x%08x 0x%08x 0x%08x\n",
+                         chid, inst << 4, nouveau_client_name(engctx), stat,
+                         type, mthd, data);
+        }
+
+        nouveau_engctx_put(engctx);
+}
+
+static void
+nv44_mpeg_me_intr(struct nouveau_subdev *subdev)
+{
+        struct nv44_mpeg_priv *priv = (void *)subdev;
+        u32 stat;
+
+        if ((stat = nv_rd32(priv, 0x00b100)))
+                nv44_mpeg_intr(subdev);
+
+        if ((stat = nv_rd32(priv, 0x00b800))) {
+                nv_error(priv, "PMSRCH 0x%08x\n", stat);
+                nv_wr32(priv, 0x00b800, stat);
+        }
+}
+
+static int
+nv44_mpeg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+               struct nouveau_oclass *oclass, void *data, u32 size,
+               struct nouveau_object **pobject)
+{
+        struct nv44_mpeg_priv *priv;
+        int ret;
+
+        ret = nouveau_mpeg_create(parent, engine, oclass, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        nv_subdev(priv)->unit = 0x00000002;
+        nv_subdev(priv)->intr = nv44_mpeg_me_intr;
+        nv_engine(priv)->cclass = &nv44_mpeg_cclass;
+        nv_engine(priv)->sclass = nv40_mpeg_sclass;
+        nv_engine(priv)->tile_prog = nv31_mpeg_tile_prog;
+        return 0;
+}
+
+struct nouveau_oclass
+nv44_mpeg_oclass = {
+        .handle = NV_ENGINE(MPEG, 0x44),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv44_mpeg_ctor,
+                .dtor = _nouveau_mpeg_dtor,
+                .init = nv31_mpeg_init,
+                .fini = _nouveau_mpeg_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/perfmon/base.c b/drivers/gpu/drm/nouveau/core/engine/perfmon/base.c
new file mode 100644
index 000000000000..e9c5e51943ef
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/perfmon/base.c
@@ -0,0 +1,449 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/option.h>
+#include <core/class.h>
+
+#include <subdev/clock.h>
+
+#include "priv.h"
+
+#define QUAD_MASK 0x0f
+#define QUAD_FREE 0x01
+
+static struct nouveau_perfsig *
+nouveau_perfsig_find_(struct nouveau_perfdom *dom, const char *name, u32 size)
+{
+        char path[64];
+        int i;
+
+        if (name[0] != '/') {
+                for (i = 0; i < dom->signal_nr; i++) {
+                        if ( dom->signal[i].name &&
+                            !strncmp(name, dom->signal[i].name, size))
+                                return &dom->signal[i];
+                }
+        } else {
+                for (i = 0; i < dom->signal_nr; i++) {
+                        snprintf(path, sizeof(path), "/%s/%02x", dom->name, i);
+                        if (!strncmp(name, path, size))
+                                return &dom->signal[i];
+                }
+        }
+
+        return NULL;
+}
+
+struct nouveau_perfsig *
+nouveau_perfsig_find(struct nouveau_perfmon *ppm, const char *name, u32 size,
+                     struct nouveau_perfdom **pdom)
+{
+        struct nouveau_perfdom *dom = *pdom;
+        struct nouveau_perfsig *sig;
+
+        if (dom == NULL) {
+                list_for_each_entry(dom, &ppm->domains, head) {
+                        sig = nouveau_perfsig_find_(dom, name, size);
+                        if (sig) {
+                                *pdom = dom;
+                                return sig;
+                        }
+                }
+
+                return NULL;
+        }
+
+        return nouveau_perfsig_find_(dom, name, size);
+}
+
+struct nouveau_perfctr *
+nouveau_perfsig_wrap(struct nouveau_perfmon *ppm, const char *name,
+                     struct nouveau_perfdom **pdom)
+{
+        struct nouveau_perfsig *sig;
+        struct nouveau_perfctr *ctr;
+
+        sig = nouveau_perfsig_find(ppm, name, strlen(name), pdom);
+        if (!sig)
+                return NULL;
+
+        ctr = kzalloc(sizeof(*ctr), GFP_KERNEL);
+        if (ctr) {
+                ctr->signal[0] = sig;
+                ctr->logic_op = 0xaaaa;
+        }
+
+        return ctr;
+}
+
+/*******************************************************************************
+ * Perfmon object classes
+ ******************************************************************************/
+static int
+nouveau_perfctr_query(struct nouveau_object *object, u32 mthd,
+                      void *data, u32 size)
+{
+        struct nouveau_device *device = nv_device(object);
+        struct nouveau_perfmon *ppm = (void *)object->engine;
+        struct nouveau_perfdom *dom = NULL, *chk;
+        struct nv_perfctr_query *args = data;
+        const bool all = nouveau_boolopt(device->cfgopt, "NvPmShowAll", false);
+        const bool raw = nouveau_boolopt(device->cfgopt, "NvPmUnnamed", all);
+        const char *name;
+        int tmp = 0, di, si;
+        char path[64];
+
+        if (size < sizeof(*args))
+                return -EINVAL;
+
+        di = (args->iter & 0xff000000) >> 24;
+        si = (args->iter & 0x00ffffff) - 1;
+
+        list_for_each_entry(chk, &ppm->domains, head) {
+                if (tmp++ == di) {
+                        dom = chk;
+                        break;
+                }
+        }
+
+        if (dom == NULL || si >= (int)dom->signal_nr)
+                return -EINVAL;
+
+        if (si >= 0) {
+                if (raw || !(name = dom->signal[si].name)) {
+                        snprintf(path, sizeof(path), "/%s/%02x", dom->name, si);
+                        name = path;
+                }
+
+                if (args->name)
+                        strncpy(args->name, name, args->size);
+                args->size = strlen(name) + 1;
+        }
+
+        do {
+                while (++si < dom->signal_nr) {
+                        if (all || dom->signal[si].name) {
+                                args->iter = (di << 24) | ++si;
+                                return 0;
+                        }
+                }
+                si = -1;
+                di = di + 1;
+                dom = list_entry(dom->head.next, typeof(*dom), head);
+        } while (&dom->head != &ppm->domains);
+
+        args->iter = 0xffffffff;
+        return 0;
+}
+
+static int
+nouveau_perfctr_sample(struct nouveau_object *object, u32 mthd,
+                       void *data, u32 size)
+{
+        struct nouveau_perfmon *ppm = (void *)object->engine;
+        struct nouveau_perfctr *ctr, *tmp;
+        struct nouveau_perfdom *dom;
+        struct nv_perfctr_sample *args = data;
+
+        if (size < sizeof(*args))
+                return -EINVAL;
+        ppm->sequence++;
+
+        list_for_each_entry(dom, &ppm->domains, head) {
+                /* sample previous batch of counters */
+                if (dom->quad != QUAD_MASK) {
+                        dom->func->next(ppm, dom);
+                        tmp = NULL;
+                        while (!list_empty(&dom->list)) {
+                                ctr = list_first_entry(&dom->list,
+                                                        typeof(*ctr), head);
+                                if (ctr->slot < 0) break;
+                                if ( tmp && tmp == ctr) break;
+                                if (!tmp) tmp = ctr;
+                                dom->func->read(ppm, dom, ctr);
+                                ctr->slot  = -1;
+                                list_move_tail(&ctr->head, &dom->list);
+                        }
+                }
+
+                dom->quad = QUAD_MASK;
+
+                /* setup next batch of counters for sampling */
+                list_for_each_entry(ctr, &dom->list, head) {
+                        ctr->slot = ffs(dom->quad) - 1;
+                        if (ctr->slot < 0)
+                                break;
+                        dom->quad &= ~(QUAD_FREE << ctr->slot);
+                        dom->func->init(ppm, dom, ctr);
+                }
+
+                if (dom->quad != QUAD_MASK)
+                        dom->func->next(ppm, dom);
+        }
+
+        return 0;
+}
+
+static int
+nouveau_perfctr_read(struct nouveau_object *object, u32 mthd,
+                     void *data, u32 size)
+{
+        struct nouveau_perfctr *ctr = (void *)object;
+        struct nv_perfctr_read *args = data;
+
+        if (size < sizeof(*args))
+                return -EINVAL;
+        if (!ctr->clk)
+                return -EAGAIN;
+
+        args->clk = ctr->clk;
+        args->ctr = ctr->ctr;
+        return 0;
+}
+
+static void
+nouveau_perfctr_dtor(struct nouveau_object *object)
+{
+        struct nouveau_perfctr *ctr = (void *)object;
+        if (ctr->head.next)
+                list_del(&ctr->head);
+        nouveau_object_destroy(&ctr->base);
+}
+
+static int
+nouveau_perfctr_ctor(struct nouveau_object *parent,
+                     struct nouveau_object *engine,
+                     struct nouveau_oclass *oclass, void *data, u32 size,
+                     struct nouveau_object **pobject)
+{
+        struct nouveau_perfmon *ppm = (void *)engine;
+        struct nouveau_perfdom *dom = NULL;
+        struct nouveau_perfsig *sig[4] = {};
+        struct nouveau_perfctr *ctr;
+        struct nv_perfctr_class *args = data;
+        int ret, i;
+
+        if (size < sizeof(*args))
+                return -EINVAL;
+
+        for (i = 0; i < ARRAY_SIZE(args->signal) && args->signal[i].name; i++) {
+                sig[i] = nouveau_perfsig_find(ppm, args->signal[i].name,
+                                              args->signal[i].size, &dom);
+                if (!sig[i])
+                        return -EINVAL;
+        }
+
+        ret = nouveau_object_create(parent, engine, oclass, 0, &ctr);
+        *pobject = nv_object(ctr);
+        if (ret)
+                return ret;
+
+        ctr->slot = -1;
+        ctr->logic_op = args->logic_op;
+        ctr->signal[0] = sig[0];
+        ctr->signal[1] = sig[1];
+        ctr->signal[2] = sig[2];
+        ctr->signal[3] = sig[3];
+        if (dom)
+                list_add_tail(&ctr->head, &dom->list);
+        return 0;
+}
+
+static struct nouveau_ofuncs
+nouveau_perfctr_ofuncs = {
+        .ctor = nouveau_perfctr_ctor,
+        .dtor = nouveau_perfctr_dtor,
+        .init = nouveau_object_init,
+        .fini = nouveau_object_fini,
+};
+
+static struct nouveau_omthds
+nouveau_perfctr_omthds[] = {
+        { NV_PERFCTR_QUERY, NV_PERFCTR_QUERY, nouveau_perfctr_query },
+        { NV_PERFCTR_SAMPLE, NV_PERFCTR_SAMPLE, nouveau_perfctr_sample },
+        { NV_PERFCTR_READ, NV_PERFCTR_READ, nouveau_perfctr_read },
+        {}
+};
+
+struct nouveau_oclass
+nouveau_perfmon_sclass[] = {
+        { .handle = NV_PERFCTR_CLASS,
+          .ofuncs = &nouveau_perfctr_ofuncs,
+          .omthds =  nouveau_perfctr_omthds,
+        },
+        {},
+};
+
+/*******************************************************************************
+ * PPM context
+ ******************************************************************************/
+static void
+nouveau_perfctx_dtor(struct nouveau_object *object)
+{
+        struct nouveau_perfmon *ppm = (void *)object->engine;
+        mutex_lock(&nv_subdev(ppm)->mutex);
+        ppm->context = NULL;
+        mutex_unlock(&nv_subdev(ppm)->mutex);
+}
+
+static int
+nouveau_perfctx_ctor(struct nouveau_object *parent,
+                     struct nouveau_object *engine,
+                     struct nouveau_oclass *oclass, void *data, u32 size,
+                     struct nouveau_object **pobject)
+{
+        struct nouveau_perfmon *ppm = (void *)engine;
+        struct nouveau_perfctx *ctx;
+        int ret;
+
+        ret = nouveau_engctx_create(parent, engine, oclass, NULL,
+                                    0, 0, 0, &ctx);
+        *pobject = nv_object(ctx);
+        if (ret)
+                return ret;
+
+        mutex_lock(&nv_subdev(ppm)->mutex);
+        if (ppm->context == NULL)
+                ppm->context = ctx;
+        mutex_unlock(&nv_subdev(ppm)->mutex);
+
+        if (ctx != ppm->context)
+                return -EBUSY;
+
+        return 0;
+}
+
+struct nouveau_oclass
+nouveau_perfmon_cclass = {
+        .handle = NV_ENGCTX(PERFMON, 0x00),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nouveau_perfctx_ctor,
+                .dtor = nouveau_perfctx_dtor,
+                .init = _nouveau_engctx_init,
+                .fini = _nouveau_engctx_fini,
+        },
+};
+
+/*******************************************************************************
+ * PPM engine/subdev functions
+ ******************************************************************************/
+int
+nouveau_perfdom_new(struct nouveau_perfmon *ppm, const char *name, u32 mask,
+                    u32 base, u32 size_unit, u32 size_domain,
+                    const struct nouveau_specdom *spec)
+{
+        const struct nouveau_specdom *sdom;
+        const struct nouveau_specsig *ssig;
+        struct nouveau_perfdom *dom;
+        int i;
+
+        for (i = 0; i == 0 || mask; i++) {
+                u32 addr = base + (i * size_unit);
+                if (i && !(mask & (1 << i)))
+                        continue;
+
+                sdom = spec;
+                while (sdom->signal_nr) {
+                        dom = kzalloc(sizeof(*dom) + sdom->signal_nr *
+                                      sizeof(*dom->signal), GFP_KERNEL);
+                        if (!dom)
+                                return -ENOMEM;
+
+                        if (mask) {
+                                snprintf(dom->name, sizeof(dom->name),
+                                         "%s/%02x/%02x", name, i,
+                                         (int)(sdom - spec));
+                        } else {
+                                snprintf(dom->name, sizeof(dom->name),
+                                         "%s/%02x", name, (int)(sdom - spec));
+                        }
+
+                        list_add_tail(&dom->head, &ppm->domains);
+                        INIT_LIST_HEAD(&dom->list);
+                        dom->func = sdom->func;
+                        dom->addr = addr;
+                        dom->quad = QUAD_MASK;
+                        dom->signal_nr = sdom->signal_nr;
+
+                        ssig = (sdom++)->signal;
+                        while (ssig->name) {
+                                dom->signal[ssig->signal].name = ssig->name;
+                                ssig++;
+                        }
+
+                        addr += size_domain;
+                }
+
+                mask &= ~(1 << i);
+        }
+
+        return 0;
+}
+
+int
+_nouveau_perfmon_fini(struct nouveau_object *object, bool suspend)
+{
+        struct nouveau_perfmon *ppm = (void *)object;
+        return nouveau_engine_fini(&ppm->base, suspend);
+}
+
+int
+_nouveau_perfmon_init(struct nouveau_object *object)
+{
+        struct nouveau_perfmon *ppm = (void *)object;
+        return nouveau_engine_init(&ppm->base);
+}
+
+void
+_nouveau_perfmon_dtor(struct nouveau_object *object)
+{
+        struct nouveau_perfmon *ppm = (void *)object;
+        struct nouveau_perfdom *dom, *tmp;
+
+        list_for_each_entry_safe(dom, tmp, &ppm->domains, head) {
+                list_del(&dom->head);
+                kfree(dom);
+        }
+
+        nouveau_engine_destroy(&ppm->base);
+}
+
+int
+nouveau_perfmon_create_(struct nouveau_object *parent,
+                        struct nouveau_object *engine,
+                        struct nouveau_oclass *oclass,
+                        int length, void **pobject)
+{
+        struct nouveau_perfmon *ppm;
+        int ret;
+
+        ret = nouveau_engine_create_(parent, engine, oclass, true, "PPM",
+                                     "perfmon", length, pobject);
+        ppm = *pobject;
+        if (ret)
+                return ret;
+
+        INIT_LIST_HEAD(&ppm->domains);
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/engine/perfmon/daemon.c b/drivers/gpu/drm/nouveau/core/engine/perfmon/daemon.c
new file mode 100644
index 000000000000..50696cc7b7d7
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/perfmon/daemon.c
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "priv.h"
+
+static void
+pwr_perfctr_init(struct nouveau_perfmon *ppm, struct nouveau_perfdom *dom,
+                 struct nouveau_perfctr *ctr)
+{
+        u32 mask = 0x00000000;
+        u32 ctrl = 0x00000001;
+        int i;
+
+        for (i = 0; i < ARRAY_SIZE(ctr->signal) && ctr->signal[i]; i++)
+                mask |= 1 << (ctr->signal[i] - dom->signal);
+
+        nv_wr32(ppm, 0x10a504 + (ctr->slot * 0x10), mask);
+        nv_wr32(ppm, 0x10a50c + (ctr->slot * 0x10), ctrl);
+        nv_wr32(ppm, 0x10a50c + (ppm->last * 0x10), 0x00000003);
+}
+
+static void
+pwr_perfctr_read(struct nouveau_perfmon *ppm, struct nouveau_perfdom *dom,
+                 struct nouveau_perfctr *ctr)
+{
+        ctr->ctr = ppm->pwr[ctr->slot];
+        ctr->clk = ppm->pwr[ppm->last];
+}
+
+static void
+pwr_perfctr_next(struct nouveau_perfmon *ppm, struct nouveau_perfdom *dom)
+{
+        int i;
+
+        for (i = 0; i <= ppm->last; i++) {
+                ppm->pwr[i] = nv_rd32(ppm, 0x10a508 + (i * 0x10));
+                nv_wr32(ppm, 0x10a508 + (i * 0x10), 0x80000000);
+        }
+}
+
+static const struct nouveau_funcdom
+pwr_perfctr_func = {
+        .init = pwr_perfctr_init,
+        .read = pwr_perfctr_read,
+        .next = pwr_perfctr_next,
+};
+
+const struct nouveau_specdom
+nva3_perfmon_pwr[] = {
+        { 0x20, (const struct nouveau_specsig[]) {
+                        { 0x00, "pwr_gr_idle" },
+                        { 0x04, "pwr_bsp_idle" },
+                        { 0x05, "pwr_vp_idle" },
+                        { 0x06, "pwr_ppp_idle" },
+                        { 0x13, "pwr_ce0_idle" },
+                        {}
+                }, &pwr_perfctr_func },
+        {}
+};
+
+const struct nouveau_specdom
+nvc0_perfmon_pwr[] = {
+        { 0x20, (const struct nouveau_specsig[]) {
+                        { 0x00, "pwr_gr_idle" },
+                        { 0x04, "pwr_bsp_idle" },
+                        { 0x05, "pwr_vp_idle" },
+                        { 0x06, "pwr_ppp_idle" },
+                        { 0x13, "pwr_ce0_idle" },
+                        { 0x14, "pwr_ce1_idle" },
+                        {}
+                }, &pwr_perfctr_func },
+        {}
+};
+
+const struct nouveau_specdom
+nve0_perfmon_pwr[] = {
+        { 0x20, (const struct nouveau_specsig[]) {
+                        { 0x00, "pwr_gr_idle" },
+                        { 0x04, "pwr_bsp_idle" },
+                        { 0x05, "pwr_vp_idle" },
+                        { 0x06, "pwr_ppp_idle" },
+                        { 0x13, "pwr_ce0_idle" },
+                        { 0x14, "pwr_ce1_idle" },
+                        { 0x15, "pwr_ce2_idle" },
+                        {}
+                }, &pwr_perfctr_func },
+        {}
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/perfmon/nv40.c b/drivers/gpu/drm/nouveau/core/engine/perfmon/nv40.c
new file mode 100644
index 000000000000..b2a10785adb1
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/perfmon/nv40.c
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv40.h"
+
+/*******************************************************************************
+ * Perfmon object classes
+ ******************************************************************************/
+
+/*******************************************************************************
+ * PPM context
+ ******************************************************************************/
+
+/*******************************************************************************
+ * PPM engine/subdev functions
+ ******************************************************************************/
+
+static void
+nv40_perfctr_init(struct nouveau_perfmon *ppm, struct nouveau_perfdom *dom,
+                  struct nouveau_perfctr *ctr)
+{
+        struct nv40_perfmon_priv *priv = (void *)ppm;
+        struct nv40_perfmon_cntr *cntr = (void *)ctr;
+        u32 log = ctr->logic_op;
+        u32 src = 0x00000000;
+        int i;
+
+        for (i = 0; i < 4 && ctr->signal[i]; i++)
+                src |= (ctr->signal[i] - dom->signal) << (i * 8);
+
+        nv_wr32(priv, 0x00a7c0 + dom->addr, 0x00000001);
+        nv_wr32(priv, 0x00a400 + dom->addr + (cntr->base.slot * 0x40), src);
+        nv_wr32(priv, 0x00a420 + dom->addr + (cntr->base.slot * 0x40), log);
+}
+
+static void
+nv40_perfctr_read(struct nouveau_perfmon *ppm, struct nouveau_perfdom *dom,
+                  struct nouveau_perfctr *ctr)
+{
+        struct nv40_perfmon_priv *priv = (void *)ppm;
+        struct nv40_perfmon_cntr *cntr = (void *)ctr;
+
+        switch (cntr->base.slot) {
+        case 0: cntr->base.ctr = nv_rd32(priv, 0x00a700 + dom->addr); break;
+        case 1: cntr->base.ctr = nv_rd32(priv, 0x00a6c0 + dom->addr); break;
+        case 2: cntr->base.ctr = nv_rd32(priv, 0x00a680 + dom->addr); break;
+        case 3: cntr->base.ctr = nv_rd32(priv, 0x00a740 + dom->addr); break;
+        }
+        cntr->base.clk = nv_rd32(priv, 0x00a600 + dom->addr);
+}
+
+static void
+nv40_perfctr_next(struct nouveau_perfmon *ppm, struct nouveau_perfdom *dom)
+{
+        struct nv40_perfmon_priv *priv = (void *)ppm;
+        if (priv->sequence != ppm->sequence) {
+                nv_wr32(priv, 0x400084, 0x00000020);
+                priv->sequence = ppm->sequence;
+        }
+}
+
+const struct nouveau_funcdom
+nv40_perfctr_func = {
+        .init = nv40_perfctr_init,
+        .read = nv40_perfctr_read,
+        .next = nv40_perfctr_next,
+};
+
+static const struct nouveau_specdom
+nv40_perfmon[] = {
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        {}
+};
+
+int
+nv40_perfmon_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+                  struct nouveau_oclass *oclass, void *data, u32 size,
+                  struct nouveau_object **pobject)
+{
+        struct nv40_perfmon_oclass *mclass = (void *)oclass;
+        struct nv40_perfmon_priv *priv;
+        int ret;
+
+        ret = nouveau_perfmon_create(parent, engine, oclass, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        ret = nouveau_perfdom_new(&priv->base, "pm", 0, 0, 0, 4, mclass->doms);
+        if (ret)
+                return ret;
+
+        nv_engine(priv)->cclass = &nouveau_perfmon_cclass;
+        nv_engine(priv)->sclass =  nouveau_perfmon_sclass;
+        return 0;
+}
+
+struct nouveau_oclass *
+nv40_perfmon_oclass = &(struct nv40_perfmon_oclass) {
+        .base.handle = NV_ENGINE(PERFMON, 0x40),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv40_perfmon_ctor,
+                .dtor = _nouveau_perfmon_dtor,
+                .init = _nouveau_perfmon_init,
+                .fini = _nouveau_perfmon_fini,
+        },
+        .doms = nv40_perfmon,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/perfmon/nv40.h b/drivers/gpu/drm/nouveau/core/engine/perfmon/nv40.h
new file mode 100644
index 000000000000..1b5792d1df14
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/perfmon/nv40.h
@@ -0,0 +1,26 @@
+#ifndef __NVKM_PM_NV40_H__
+#define __NVKM_PM_NV40_H__
+
+#include "priv.h"
+
+struct nv40_perfmon_oclass {
+        struct nouveau_oclass base;
+        const struct nouveau_specdom *doms;
+};
+
+struct nv40_perfmon_priv {
+        struct nouveau_perfmon base;
+        u32 sequence;
+};
+
+int nv40_perfmon_ctor(struct nouveau_object *, struct nouveau_object *,
+                      struct nouveau_oclass *, void *data, u32 size,
+                      struct nouveau_object **pobject);
+
+struct nv40_perfmon_cntr {
+        struct nouveau_perfctr base;
+};
+
+extern const struct nouveau_funcdom nv40_perfctr_func;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/perfmon/nv50.c b/drivers/gpu/drm/nouveau/core/engine/perfmon/nv50.c
new file mode 100644
index 000000000000..94217691fe67
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/perfmon/nv50.c
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv40.h"
+
+/*******************************************************************************
+ * Perfmon object classes
+ ******************************************************************************/
+
+/*******************************************************************************
+ * PPM context
+ ******************************************************************************/
+
+/*******************************************************************************
+ * PPM engine/subdev functions
+ ******************************************************************************/
+
+static const struct nouveau_specdom
+nv50_perfmon[] = {
+        { 0x040, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x100, (const struct nouveau_specsig[]) {
+                        { 0xc8, "gr_idle" },
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x100, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x020, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x040, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        {}
+};
+
+struct nouveau_oclass *
+nv50_perfmon_oclass = &(struct nv40_perfmon_oclass) {
+        .base.handle = NV_ENGINE(PERFMON, 0x50),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv40_perfmon_ctor,
+                .dtor = _nouveau_perfmon_dtor,
+                .init = _nouveau_perfmon_init,
+                .fini = _nouveau_perfmon_fini,
+        },
+        .doms = nv50_perfmon,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/perfmon/nv84.c b/drivers/gpu/drm/nouveau/core/engine/perfmon/nv84.c
new file mode 100644
index 000000000000..9232c7fc6253
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/perfmon/nv84.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv40.h"
+
+/*******************************************************************************
+ * Perfmon object classes
+ ******************************************************************************/
+
+/*******************************************************************************
+ * PPM context
+ ******************************************************************************/
+
+/*******************************************************************************
+ * PPM engine/subdev functions
+ ******************************************************************************/
+
+static const struct nouveau_specdom
+nv84_perfmon[] = {
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        {}
+};
+
+struct nouveau_oclass *
+nv84_perfmon_oclass = &(struct nv40_perfmon_oclass) {
+        .base.handle = NV_ENGINE(PERFMON, 0x84),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv40_perfmon_ctor,
+                .dtor = _nouveau_perfmon_dtor,
+                .init = _nouveau_perfmon_init,
+                .fini = _nouveau_perfmon_fini,
+        },
+        .doms = nv84_perfmon,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/perfmon/nva3.c b/drivers/gpu/drm/nouveau/core/engine/perfmon/nva3.c
new file mode 100644
index 000000000000..6197ebdeb648
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/perfmon/nva3.c
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv40.h"
+
+/*******************************************************************************
+ * Perfmon object classes
+ ******************************************************************************/
+
+/*******************************************************************************
+ * PPM context
+ ******************************************************************************/
+
+/*******************************************************************************
+ * PPM engine/subdev functions
+ ******************************************************************************/
+
+static const struct nouveau_specdom
+nva3_perfmon[] = {
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        { 0x20, (const struct nouveau_specsig[]) {
+                        {}
+                }, &nv40_perfctr_func },
+        {}
+};
+
+static int
+nva3_perfmon_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+                  struct nouveau_oclass *oclass, void *data, u32 size,
+                  struct nouveau_object **object)
+{
+        int ret = nv40_perfmon_ctor(parent, engine, oclass, data, size, object);
+        if (ret == 0) {
+                struct nv40_perfmon_priv *priv = (void *)*object;
+                ret = nouveau_perfdom_new(&priv->base, "pwr", 0, 0, 0, 0,
+                                           nva3_perfmon_pwr);
+                if (ret)
+                        return ret;
+
+                priv->base.last = 3;
+        }
+        return ret;
+}
+
+struct nouveau_oclass *
+nva3_perfmon_oclass = &(struct nv40_perfmon_oclass) {
+        .base.handle = NV_ENGINE(PERFMON, 0xa3),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nva3_perfmon_ctor,
+                .dtor = _nouveau_perfmon_dtor,
+                .init = _nouveau_perfmon_init,
+                .fini = _nouveau_perfmon_fini,
+        },
+        .doms = nva3_perfmon,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/perfmon/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/perfmon/nvc0.c
new file mode 100644
index 000000000000..74b241042502
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/perfmon/nvc0.c
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nvc0.h"
+
+/*******************************************************************************
+ * Perfmon object classes
+ ******************************************************************************/
+
+/*******************************************************************************
+ * PPM context
+ ******************************************************************************/
+
+/*******************************************************************************
+ * PPM engine/subdev functions
+ ******************************************************************************/
+
+static const struct nouveau_specdom
+nvc0_perfmon_hub[] = {
+        {}
+};
+
+static const struct nouveau_specdom
+nvc0_perfmon_gpc[] = {
+        {}
+};
+
+static const struct nouveau_specdom
+nvc0_perfmon_part[] = {
+        {}
+};
+
+static void
+nvc0_perfctr_init(struct nouveau_perfmon *ppm, struct nouveau_perfdom *dom,
+                  struct nouveau_perfctr *ctr)
+{
+        struct nvc0_perfmon_priv *priv = (void *)ppm;
+        struct nvc0_perfmon_cntr *cntr = (void *)ctr;
+        u32 log = ctr->logic_op;
+        u32 src = 0x00000000;
+        int i;
+
+        for (i = 0; i < 4 && ctr->signal[i]; i++)
+                src |= (ctr->signal[i] - dom->signal) << (i * 8);
+
+        nv_wr32(priv, dom->addr + 0x09c, 0x00040002);
+        nv_wr32(priv, dom->addr + 0x100, 0x00000000);
+        nv_wr32(priv, dom->addr + 0x040 + (cntr->base.slot * 0x08), src);
+        nv_wr32(priv, dom->addr + 0x044 + (cntr->base.slot * 0x08), log);
+}
+
+static void
+nvc0_perfctr_read(struct nouveau_perfmon *ppm, struct nouveau_perfdom *dom,
+                  struct nouveau_perfctr *ctr)
+{
+        struct nvc0_perfmon_priv *priv = (void *)ppm;
+        struct nvc0_perfmon_cntr *cntr = (void *)ctr;
+
+        switch (cntr->base.slot) {
+        case 0: cntr->base.ctr = nv_rd32(priv, dom->addr + 0x08c); break;
+        case 1: cntr->base.ctr = nv_rd32(priv, dom->addr + 0x088); break;
+        case 2: cntr->base.ctr = nv_rd32(priv, dom->addr + 0x080); break;
+        case 3: cntr->base.ctr = nv_rd32(priv, dom->addr + 0x090); break;
+        }
+        cntr->base.clk = nv_rd32(priv, dom->addr + 0x070);
+}
+
+static void
+nvc0_perfctr_next(struct nouveau_perfmon *ppm, struct nouveau_perfdom *dom)
+{
+        struct nvc0_perfmon_priv *priv = (void *)ppm;
+        nv_wr32(priv, dom->addr + 0x06c, dom->signal_nr - 0x40 + 0x27);
+        nv_wr32(priv, dom->addr + 0x0ec, 0x00000011);
+}
+
+const struct nouveau_funcdom
+nvc0_perfctr_func = {
+        .init = nvc0_perfctr_init,
+        .read = nvc0_perfctr_read,
+        .next = nvc0_perfctr_next,
+};
+
+int
+nvc0_perfmon_fini(struct nouveau_object *object, bool suspend)
+{
+        struct nvc0_perfmon_priv *priv = (void *)object;
+        nv_mask(priv, 0x000200, 0x10000000, 0x00000000);
+        nv_mask(priv, 0x000200, 0x10000000, 0x10000000);
+        return nouveau_perfmon_fini(&priv->base, suspend);
+}
+
+static int
+nvc0_perfmon_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+                  struct nouveau_oclass *oclass, void *data, u32 size,
+                  struct nouveau_object **pobject)
+{
+        struct nvc0_perfmon_priv *priv;
+        u32 mask;
+        int ret;
+
+        ret = nouveau_perfmon_create(parent, engine, oclass, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        ret = nouveau_perfdom_new(&priv->base, "pwr", 0, 0, 0, 0,
+                                   nvc0_perfmon_pwr);
+        if (ret)
+                return ret;
+
+        /* HUB */
+        ret = nouveau_perfdom_new(&priv->base, "hub", 0, 0x1b0000, 0, 0x200,
+                                   nvc0_perfmon_hub);
+        if (ret)
+                return ret;
+
+        /* GPC */
+        mask  = (1 << nv_rd32(priv, 0x022430)) - 1;
+        mask &= ~nv_rd32(priv, 0x022504);
+        mask &= ~nv_rd32(priv, 0x022584);
+
+        ret = nouveau_perfdom_new(&priv->base, "gpc", mask, 0x180000,
+                                  0x1000, 0x200, nvc0_perfmon_gpc);
+        if (ret)
+                return ret;
+
+        /* PART */
+        mask  = (1 << nv_rd32(priv, 0x022438)) - 1;
+        mask &= ~nv_rd32(priv, 0x022548);
+        mask &= ~nv_rd32(priv, 0x0225c8);
+
+        ret = nouveau_perfdom_new(&priv->base, "part", mask, 0x1a0000,
+                                  0x1000, 0x200, nvc0_perfmon_part);
+        if (ret)
+                return ret;
+
+        nv_engine(priv)->cclass = &nouveau_perfmon_cclass;
+        nv_engine(priv)->sclass =  nouveau_perfmon_sclass;
+        priv->base.last = 7;
+        return 0;
+}
+
+struct nouveau_oclass
+nvc0_perfmon_oclass = {
+        .handle = NV_ENGINE(PERFMON, 0xc0),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nvc0_perfmon_ctor,
+                .dtor = _nouveau_perfmon_dtor,
+                .init = _nouveau_perfmon_init,
+                .fini = nvc0_perfmon_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/perfmon/nvc0.h b/drivers/gpu/drm/nouveau/core/engine/perfmon/nvc0.h
new file mode 100644
index 000000000000..f66bca484263
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/perfmon/nvc0.h
@@ -0,0 +1,17 @@
+#ifndef __NVKM_PM_NVC0_H__
+#define __NVKM_PM_NVC0_H__
+
+#include "priv.h"
+
+struct nvc0_perfmon_priv {
+        struct nouveau_perfmon base;
+};
+
+struct nvc0_perfmon_cntr {
+        struct nouveau_perfctr base;
+};
+
+extern const struct nouveau_funcdom nvc0_perfctr_func;
+int nvc0_perfmon_fini(struct nouveau_object *, bool);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/perfmon/nve0.c b/drivers/gpu/drm/nouveau/core/engine/perfmon/nve0.c
new file mode 100644
index 000000000000..71d718c12075
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/perfmon/nve0.c
@@ -0,0 +1,162 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nvc0.h"
+
+/*******************************************************************************
+ * Perfmon object classes
+ ******************************************************************************/
+
+/*******************************************************************************
+ * PPM context
+ ******************************************************************************/
+
+/*******************************************************************************
+ * PPM engine/subdev functions
+ ******************************************************************************/
+
+static const struct nouveau_specdom
+nve0_perfmon_hub[] = {
+        { 0x60, (const struct nouveau_specsig[]) {
+                        { 0x47, "hub00_user_0" },
+                        {}
+                }, &nvc0_perfctr_func },
+        { 0x40, (const struct nouveau_specsig[]) {
+                        { 0x27, "hub01_user_0" },
+                        {}
+                }, &nvc0_perfctr_func },
+        { 0x60, (const struct nouveau_specsig[]) {
+                        { 0x47, "hub02_user_0" },
+                        {}
+                }, &nvc0_perfctr_func },
+        { 0x60, (const struct nouveau_specsig[]) {
+                        { 0x47, "hub03_user_0" },
+                        {}
+                }, &nvc0_perfctr_func },
+        { 0x40, (const struct nouveau_specsig[]) {
+                        { 0x03, "host_mmio_rd" },
+                        { 0x27, "hub04_user_0" },
+                        {}
+                }, &nvc0_perfctr_func },
+        { 0x60, (const struct nouveau_specsig[]) {
+                        { 0x47, "hub05_user_0" },
+                        {}
+                }, &nvc0_perfctr_func },
+        { 0xc0, (const struct nouveau_specsig[]) {
+                        { 0x74, "host_fb_rd3x" },
+                        { 0x75, "host_fb_rd3x_2" },
+                        { 0xa7, "hub06_user_0" },
+                        {}
+                }, &nvc0_perfctr_func },
+        { 0x60, (const struct nouveau_specsig[]) {
+                        { 0x47, "hub07_user_0" },
+                        {}
+                }, &nvc0_perfctr_func },
+        {}
+};
+
+static const struct nouveau_specdom
+nve0_perfmon_gpc[] = {
+        { 0xe0, (const struct nouveau_specsig[]) {
+                        { 0xc7, "gpc00_user_0" },
+                        {}
+                }, &nvc0_perfctr_func },
+        {}
+};
+
+static const struct nouveau_specdom
+nve0_perfmon_part[] = {
+        { 0x60, (const struct nouveau_specsig[]) {
+                        { 0x47, "part00_user_0" },
+                        {}
+                }, &nvc0_perfctr_func },
+        { 0x60, (const struct nouveau_specsig[]) {
+                        { 0x47, "part01_user_0" },
+                        {}
+                }, &nvc0_perfctr_func },
+        {}
+};
+
+static int
+nve0_perfmon_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+                  struct nouveau_oclass *oclass, void *data, u32 size,
+                  struct nouveau_object **pobject)
+{
+        struct nvc0_perfmon_priv *priv;
+        u32 mask;
+        int ret;
+
+        ret = nouveau_perfmon_create(parent, engine, oclass, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        /* PDAEMON */
+        ret = nouveau_perfdom_new(&priv->base, "pwr", 0, 0, 0, 0,
+                                   nve0_perfmon_pwr);
+        if (ret)
+                return ret;
+
+        /* HUB */
+        ret = nouveau_perfdom_new(&priv->base, "hub", 0, 0x1b0000, 0, 0x200,
+                                   nve0_perfmon_hub);
+        if (ret)
+                return ret;
+
+        /* GPC */
+        mask  = (1 << nv_rd32(priv, 0x022430)) - 1;
+        mask &= ~nv_rd32(priv, 0x022504);
+        mask &= ~nv_rd32(priv, 0x022584);
+
+        ret = nouveau_perfdom_new(&priv->base, "gpc", mask, 0x180000,
+                                  0x1000, 0x200, nve0_perfmon_gpc);
+        if (ret)
+                return ret;
+
+        /* PART */
+        mask  = (1 << nv_rd32(priv, 0x022438)) - 1;
+        mask &= ~nv_rd32(priv, 0x022548);
+        mask &= ~nv_rd32(priv, 0x0225c8);
+
+        ret = nouveau_perfdom_new(&priv->base, "part", mask, 0x1a0000,
+                                  0x1000, 0x200, nve0_perfmon_part);
+        if (ret)
+                return ret;
+
+        nv_engine(priv)->cclass = &nouveau_perfmon_cclass;
+        nv_engine(priv)->sclass =  nouveau_perfmon_sclass;
+        priv->base.last = 7;
+        return 0;
+}
+
+struct nouveau_oclass
+nve0_perfmon_oclass = {
+        .handle = NV_ENGINE(PERFMON, 0xe0),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nve0_perfmon_ctor,
+                .dtor = _nouveau_perfmon_dtor,
+                .init = _nouveau_perfmon_init,
+                .fini = nvc0_perfmon_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/perfmon/nvf0.c b/drivers/gpu/drm/nouveau/core/engine/perfmon/nvf0.c
new file mode 100644
index 000000000000..47256f78a895
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/perfmon/nvf0.c
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nvc0.h"
+
+/*******************************************************************************
+ * Perfmon object classes
+ ******************************************************************************/
+
+/*******************************************************************************
+ * PPM context
+ ******************************************************************************/
+
+/*******************************************************************************
+ * PPM engine/subdev functions
+ ******************************************************************************/
+
+static int
+nvf0_perfmon_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+                  struct nouveau_oclass *oclass, void *data, u32 size,
+                  struct nouveau_object **pobject)
+{
+        struct nvc0_perfmon_priv *priv;
+        int ret;
+
+        ret = nouveau_perfmon_create(parent, engine, oclass, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        ret = nouveau_perfdom_new(&priv->base, "pwr", 0, 0, 0, 0,
+                                   nve0_perfmon_pwr);
+        if (ret)
+                return ret;
+
+        nv_engine(priv)->cclass = &nouveau_perfmon_cclass;
+        nv_engine(priv)->sclass =  nouveau_perfmon_sclass;
+        return 0;
+}
+
+struct nouveau_oclass
+nvf0_perfmon_oclass = {
+        .handle = NV_ENGINE(PERFMON, 0xf0),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nvf0_perfmon_ctor,
+                .dtor = _nouveau_perfmon_dtor,
+                .init = _nouveau_perfmon_init,
+                .fini = nvc0_perfmon_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/perfmon/priv.h b/drivers/gpu/drm/nouveau/core/engine/perfmon/priv.h
new file mode 100644
index 000000000000..0ac8714fe0ba
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/perfmon/priv.h
@@ -0,0 +1,91 @@
+#ifndef __NVKM_PERFMON_PRIV_H__
+#define __NVKM_PERFMON_PRIV_H__
+
+#include <engine/perfmon.h>
+
+struct nouveau_perfctr {
+        struct nouveau_object base;
+        struct list_head head;
+        struct nouveau_perfsig *signal[4];
+        int slot;
+        u32 logic_op;
+        u32 clk;
+        u32 ctr;
+};
+
+extern struct nouveau_oclass nouveau_perfmon_sclass[];
+
+struct nouveau_perfctx {
+        struct nouveau_engctx base;
+};
+
+extern struct nouveau_oclass nouveau_perfmon_cclass;
+
+struct nouveau_specsig {
+        u8 signal;
+        const char *name;
+};
+
+struct nouveau_perfsig {
+        const char *name;
+};
+
+struct nouveau_perfdom;
+struct nouveau_perfctr *
+nouveau_perfsig_wrap(struct nouveau_perfmon *, const char *,
+                     struct nouveau_perfdom **);
+
+struct nouveau_specdom {
+        u16 signal_nr;
+        const struct nouveau_specsig *signal;
+        const struct nouveau_funcdom *func;
+};
+
+extern const struct nouveau_specdom nva3_perfmon_pwr[];
+extern const struct nouveau_specdom nvc0_perfmon_pwr[];
+extern const struct nouveau_specdom nve0_perfmon_pwr[];
+
+struct nouveau_perfdom {
+        struct list_head head;
+        struct list_head list;
+        const struct nouveau_funcdom *func;
+        char name[32];
+        u32 addr;
+        u8  quad;
+        u32 signal_nr;
+        struct nouveau_perfsig signal[];
+};
+
+struct nouveau_funcdom {
+        void (*init)(struct nouveau_perfmon *, struct nouveau_perfdom *,
+                     struct nouveau_perfctr *);
+        void (*read)(struct nouveau_perfmon *, struct nouveau_perfdom *,
+                     struct nouveau_perfctr *);
+        void (*next)(struct nouveau_perfmon *, struct nouveau_perfdom *);
+};
+
+int nouveau_perfdom_new(struct nouveau_perfmon *, const char *, u32,
+                        u32, u32, u32, const struct nouveau_specdom *);
+
+#define nouveau_perfmon_create(p,e,o,d)                                        \
+        nouveau_perfmon_create_((p), (e), (o), sizeof(**d), (void **)d)
+#define nouveau_perfmon_dtor(p) ({                                             \
+        struct nouveau_perfmon *c = (p);                                       \
+        _nouveau_perfmon_dtor(nv_object(c));                                   \
+})
+#define nouveau_perfmon_init(p) ({                                             \
+        struct nouveau_perfmon *c = (p);                                       \
+        _nouveau_perfmon_init(nv_object(c));                                   \
+})
+#define nouveau_perfmon_fini(p,s) ({                                           \
+        struct nouveau_perfmon *c = (p);                                       \
+        _nouveau_perfmon_fini(nv_object(c), (s));                              \
+})
+
+int nouveau_perfmon_create_(struct nouveau_object *, struct nouveau_object *,
+                            struct nouveau_oclass *, int, void **);
+void _nouveau_perfmon_dtor(struct nouveau_object *);
+int  _nouveau_perfmon_init(struct nouveau_object *);
+int  _nouveau_perfmon_fini(struct nouveau_object *, bool);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/software/nv04.c b/drivers/gpu/drm/nouveau/core/engine/software/nv04.c
index 2a859a31c30d..c571758e4a27 100644
--- a/drivers/gpu/drm/nouveau/core/engine/software/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/engine/software/nv04.c
@@ -135,8 +135,8 @@ nv04_software_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-struct nouveau_oclass
-nv04_software_oclass = {
+struct nouveau_oclass *
+nv04_software_oclass = &(struct nouveau_oclass) {
         .handle = NV_ENGINE(SW, 0x04),
         .ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv04_software_ctor,
diff --git a/drivers/gpu/drm/nouveau/core/engine/software/nv10.c b/drivers/gpu/drm/nouveau/core/engine/software/nv10.c
index a019364b1e13..a62f11a78430 100644
--- a/drivers/gpu/drm/nouveau/core/engine/software/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/engine/software/nv10.c
@@ -117,8 +117,8 @@ nv10_software_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-struct nouveau_oclass
-nv10_software_oclass = {
+struct nouveau_oclass *
+nv10_software_oclass = &(struct nouveau_oclass) {
         .handle = NV_ENGINE(SW, 0x10),
         .ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv10_software_ctor,
diff --git a/drivers/gpu/drm/nouveau/core/engine/software/nv50.c b/drivers/gpu/drm/nouveau/core/engine/software/nv50.c
index c48e74953771..b574dd4bb828 100644
--- a/drivers/gpu/drm/nouveau/core/engine/software/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/software/nv50.c
@@ -32,16 +32,9 @@
 
 #include <subdev/bar.h>
 
-#include <engine/software.h>
 #include <engine/disp.h>
 
-struct nv50_software_priv {
-        struct nouveau_software base;
-};
-
-struct nv50_software_chan {
-        struct nouveau_software_chan base;
-};
+#include "nv50.h"
 
 /*******************************************************************************
  * software object classes
@@ -62,7 +55,7 @@ nv50_software_mthd_dma_vblsem(struct nouveau_object *object, u32 mthd,
 
         if (nv_iclass(handle->object, NV_GPUOBJ_CLASS)) {
                 struct nouveau_gpuobj *gpuobj = nv_gpuobj(handle->object);
-                chan->base.vblank.ctxdma = gpuobj->node->offset >> 4;
+                chan->vblank.ctxdma = gpuobj->node->offset >> 4;
                 ret = 0;
         }
         nouveau_namedb_put(handle);
@@ -74,34 +67,33 @@ nv50_software_mthd_vblsem_offset(struct nouveau_object *object, u32 mthd,
                                  void *args, u32 size)
 {
         struct nv50_software_chan *chan = (void *)nv_engctx(object->parent);
-        chan->base.vblank.offset = *(u32 *)args;
+        chan->vblank.offset = *(u32 *)args;
         return 0;
 }
 
-static int
+int
 nv50_software_mthd_vblsem_value(struct nouveau_object *object, u32 mthd,
                                 void *args, u32 size)
 {
         struct nv50_software_chan *chan = (void *)nv_engctx(object->parent);
-        chan->base.vblank.value = *(u32 *)args;
+        chan->vblank.value = *(u32 *)args;
         return 0;
 }
 
-static int
+int
 nv50_software_mthd_vblsem_release(struct nouveau_object *object, u32 mthd,
                                   void *args, u32 size)
 {
         struct nv50_software_chan *chan = (void *)nv_engctx(object->parent);
-        struct nouveau_disp *disp = nouveau_disp(object);
-        u32 crtc = *(u32 *)args;
-        if (crtc > 1)
+        u32 head = *(u32 *)args;
+        if (head >= chan->vblank.nr_event)
                 return -EINVAL;
 
-        nouveau_event_get(disp->vblank, crtc, &chan->base.vblank.event);
+        nouveau_event_get(chan->vblank.event[head]);
         return 0;
 }
 
-static int
+int
 nv50_software_mthd_flip(struct nouveau_object *object, u32 mthd,
                         void *args, u32 size)
 {
@@ -132,10 +124,9 @@ nv50_software_sclass[] = {
  ******************************************************************************/
 
 static int
-nv50_software_vblsem_release(struct nouveau_eventh *event, int head)
+nv50_software_vblsem_release(void *data, int head)
 {
-        struct nouveau_software_chan *chan =
-                container_of(event, struct nouveau_software_chan, vblank.event);
+        struct nv50_software_chan *chan = data;
         struct nv50_software_priv *priv = (void *)nv_object(chan)->engine;
         struct nouveau_bar *bar = nouveau_bar(priv);
 
@@ -154,45 +145,76 @@ nv50_software_vblsem_release(struct nouveau_eventh *event, int head)
         return NVKM_EVENT_DROP;
 }
 
-static int
+void
+nv50_software_context_dtor(struct nouveau_object *object)
+{
+        struct nv50_software_chan *chan = (void *)object;
+        int i;
+
+        if (chan->vblank.event) {
+                for (i = 0; i < chan->vblank.nr_event; i++)
+                        nouveau_event_ref(NULL, &chan->vblank.event[i]);
+                kfree(chan->vblank.event);
+        }
+
+        nouveau_software_context_destroy(&chan->base);
+}
+
+int
 nv50_software_context_ctor(struct nouveau_object *parent,
                            struct nouveau_object *engine,
                            struct nouveau_oclass *oclass, void *data, u32 size,
                            struct nouveau_object **pobject)
 {
+        struct nouveau_disp *pdisp = nouveau_disp(parent);
+        struct nv50_software_cclass *pclass = (void *)oclass;
         struct nv50_software_chan *chan;
-        int ret;
+        int ret, i;
 
         ret = nouveau_software_context_create(parent, engine, oclass, &chan);
         *pobject = nv_object(chan);
         if (ret)
                 return ret;
 
-        chan->base.vblank.channel = nv_gpuobj(parent->parent)->addr >> 12;
-        chan->base.vblank.event.func = nv50_software_vblsem_release;
+        chan->vblank.nr_event = pdisp->vblank->index_nr;
+        chan->vblank.event = kzalloc(chan->vblank.nr_event *
+                                     sizeof(*chan->vblank.event), GFP_KERNEL);
+        if (!chan->vblank.event)
+                return -ENOMEM;
+
+        for (i = 0; i < chan->vblank.nr_event; i++) {
+                ret = nouveau_event_new(pdisp->vblank, i, pclass->vblank,
+                                        chan, &chan->vblank.event[i]);
+                if (ret)
+                        return ret;
+        }
+
+        chan->vblank.channel = nv_gpuobj(parent->parent)->addr >> 12;
         return 0;
 }
 
-static struct nouveau_oclass
+static struct nv50_software_cclass
 nv50_software_cclass = {
-        .handle = NV_ENGCTX(SW, 0x50),
-        .ofuncs = &(struct nouveau_ofuncs) {
+        .base.handle = NV_ENGCTX(SW, 0x50),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv50_software_context_ctor,
                 .dtor = _nouveau_software_context_dtor,
                 .init = _nouveau_software_context_init,
                 .fini = _nouveau_software_context_fini,
         },
+        .vblank = nv50_software_vblsem_release,
 };
 
 /*******************************************************************************
  * software engine/subdev functions
  ******************************************************************************/
 
-static int
+int
 nv50_software_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                    struct nouveau_oclass *oclass, void *data, u32 size,
                    struct nouveau_object **pobject)
 {
+        struct nv50_software_oclass *pclass = (void *)oclass;
         struct nv50_software_priv *priv;
         int ret;
 
@@ -201,19 +223,21 @@ nv50_software_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        nv_engine(priv)->cclass = &nv50_software_cclass;
-        nv_engine(priv)->sclass = nv50_software_sclass;
+        nv_engine(priv)->cclass = pclass->cclass;
+        nv_engine(priv)->sclass = pclass->sclass;
         nv_subdev(priv)->intr = nv04_software_intr;
         return 0;
 }
 
-struct nouveau_oclass
-nv50_software_oclass = {
-        .handle = NV_ENGINE(SW, 0x50),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nv50_software_oclass = &(struct nv50_software_oclass) {
+        .base.handle = NV_ENGINE(SW, 0x50),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv50_software_ctor,
                 .dtor = _nouveau_software_dtor,
                 .init = _nouveau_software_init,
                 .fini = _nouveau_software_fini,
         },
-};
+        .cclass = &nv50_software_cclass.base,
+        .sclass =  nv50_software_sclass,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/software/nv50.h b/drivers/gpu/drm/nouveau/core/engine/software/nv50.h
new file mode 100644
index 000000000000..2de370c21279
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/software/nv50.h
@@ -0,0 +1,47 @@
+#ifndef __NVKM_SW_NV50_H__
+#define __NVKM_SW_NV50_H__
+
+#include <engine/software.h>
+
+struct nv50_software_oclass {
+        struct nouveau_oclass base;
+        struct nouveau_oclass *cclass;
+        struct nouveau_oclass *sclass;
+};
+
+struct nv50_software_priv {
+        struct nouveau_software base;
+};
+
+int  nv50_software_ctor(struct nouveau_object *, struct nouveau_object *,
+                        struct nouveau_oclass *, void *, u32,
+                        struct nouveau_object **);
+
+struct nv50_software_cclass {
+        struct nouveau_oclass base;
+        int (*vblank)(void *, int);
+};
+
+struct nv50_software_chan {
+        struct nouveau_software_chan base;
+        struct {
+                struct nouveau_eventh **event;
+                int nr_event;
+                u32 channel;
+                u32 ctxdma;
+                u64 offset;
+                u32 value;
+        } vblank;
+};
+
+int  nv50_software_context_ctor(struct nouveau_object *,
+                                struct nouveau_object *,
+                                struct nouveau_oclass *, void *, u32,
+                                struct nouveau_object **);
+void nv50_software_context_dtor(struct nouveau_object *);
+
+int nv50_software_mthd_vblsem_value(struct nouveau_object *, u32, void *, u32);
+int nv50_software_mthd_vblsem_release(struct nouveau_object *, u32, void *, u32);
+int nv50_software_mthd_flip(struct nouveau_object *, u32, void *, u32);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/software/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/software/nvc0.c
index d698e710ddd4..f9430c1bf3e5 100644
--- a/drivers/gpu/drm/nouveau/core/engine/software/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/software/nvc0.c
@@ -32,13 +32,7 @@
 #include <engine/software.h>
 #include <engine/disp.h>
 
-struct nvc0_software_priv {
-        struct nouveau_software base;
-};
-
-struct nvc0_software_chan {
-        struct nouveau_software_chan base;
-};
+#include "nv50.h"
 
 /*******************************************************************************
  * software object classes
@@ -48,58 +42,24 @@ static int
 nvc0_software_mthd_vblsem_offset(struct nouveau_object *object, u32 mthd,
                                  void *args, u32 size)
 {
-        struct nvc0_software_chan *chan = (void *)nv_engctx(object->parent);
+        struct nv50_software_chan *chan = (void *)nv_engctx(object->parent);
         u64 data = *(u32 *)args;
         if (mthd == 0x0400) {
-                chan->base.vblank.offset &= 0x00ffffffffULL;
-                chan->base.vblank.offset |= data << 32;
+                chan->vblank.offset &= 0x00ffffffffULL;
+                chan->vblank.offset |= data << 32;
         } else {
-                chan->base.vblank.offset &= 0xff00000000ULL;
-                chan->base.vblank.offset |= data;
+                chan->vblank.offset &= 0xff00000000ULL;
+                chan->vblank.offset |= data;
         }
         return 0;
 }
 
 static int
-nvc0_software_mthd_vblsem_value(struct nouveau_object *object, u32 mthd,
-                                void *args, u32 size)
-{
-        struct nvc0_software_chan *chan = (void *)nv_engctx(object->parent);
-        chan->base.vblank.value = *(u32 *)args;
-        return 0;
-}
-
-static int
-nvc0_software_mthd_vblsem_release(struct nouveau_object *object, u32 mthd,
-                                  void *args, u32 size)
-{
-        struct nvc0_software_chan *chan = (void *)nv_engctx(object->parent);
-        struct nouveau_disp *disp = nouveau_disp(object);
-        u32 crtc = *(u32 *)args;
-
-        if ((nv_device(object)->card_type < NV_E0 && crtc > 1) || crtc > 3)
-                return -EINVAL;
-
-        nouveau_event_get(disp->vblank, crtc, &chan->base.vblank.event);
-        return 0;
-}
-
-static int
-nvc0_software_mthd_flip(struct nouveau_object *object, u32 mthd,
-                        void *args, u32 size)
-{
-        struct nvc0_software_chan *chan = (void *)nv_engctx(object->parent);
-        if (chan->base.flip)
-                return chan->base.flip(chan->base.flip_data);
-        return -EINVAL;
-}
-
-static int
 nvc0_software_mthd_mp_control(struct nouveau_object *object, u32 mthd,
                               void *args, u32 size)
 {
-        struct nvc0_software_chan *chan = (void *)nv_engctx(object->parent);
-        struct nvc0_software_priv *priv = (void *)nv_object(chan)->engine;
+        struct nv50_software_chan *chan = (void *)nv_engctx(object->parent);
+        struct nv50_software_priv *priv = (void *)nv_object(chan)->engine;
         u32 data = *(u32 *)args;
 
         switch (mthd) {
@@ -124,9 +84,9 @@ static struct nouveau_omthds
 nvc0_software_omthds[] = {
         { 0x0400, 0x0400, nvc0_software_mthd_vblsem_offset },
         { 0x0404, 0x0404, nvc0_software_mthd_vblsem_offset },
-        { 0x0408, 0x0408, nvc0_software_mthd_vblsem_value },
-        { 0x040c, 0x040c, nvc0_software_mthd_vblsem_release },
-        { 0x0500, 0x0500, nvc0_software_mthd_flip },
+        { 0x0408, 0x0408, nv50_software_mthd_vblsem_value },
+        { 0x040c, 0x040c, nv50_software_mthd_vblsem_release },
+        { 0x0500, 0x0500, nv50_software_mthd_flip },
         { 0x0600, 0x0600, nvc0_software_mthd_mp_control },
         { 0x0644, 0x0644, nvc0_software_mthd_mp_control },
         { 0x06ac, 0x06ac, nvc0_software_mthd_mp_control },
@@ -144,11 +104,10 @@ nvc0_software_sclass[] = {
  ******************************************************************************/
 
 static int
-nvc0_software_vblsem_release(struct nouveau_eventh *event, int head)
+nvc0_software_vblsem_release(void *data, int head)
 {
-        struct nouveau_software_chan *chan =
-                container_of(event, struct nouveau_software_chan, vblank.event);
-        struct nvc0_software_priv *priv = (void *)nv_object(chan)->engine;
+        struct nv50_software_chan *chan = data;
+        struct nv50_software_priv *priv = (void *)nv_object(chan)->engine;
         struct nouveau_bar *bar = nouveau_bar(priv);
 
         nv_wr32(priv, 0x001718, 0x80000000 | chan->vblank.channel);
@@ -160,66 +119,31 @@ nvc0_software_vblsem_release(struct nouveau_eventh *event, int head)
         return NVKM_EVENT_DROP;
 }
 
-static int
-nvc0_software_context_ctor(struct nouveau_object *parent,
-                           struct nouveau_object *engine,
-                           struct nouveau_oclass *oclass, void *data, u32 size,
-                           struct nouveau_object **pobject)
-{
-        struct nvc0_software_chan *chan;
-        int ret;
-
-        ret = nouveau_software_context_create(parent, engine, oclass, &chan);
-        *pobject = nv_object(chan);
-        if (ret)
-                return ret;
-
-        chan->base.vblank.channel = nv_gpuobj(parent->parent)->addr >> 12;
-        chan->base.vblank.event.func = nvc0_software_vblsem_release;
-        return 0;
-}
-
-static struct nouveau_oclass
+static struct nv50_software_cclass
 nvc0_software_cclass = {
-        .handle = NV_ENGCTX(SW, 0xc0),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nvc0_software_context_ctor,
+        .base.handle = NV_ENGCTX(SW, 0xc0),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv50_software_context_ctor,
                 .dtor = _nouveau_software_context_dtor,
                 .init = _nouveau_software_context_init,
                 .fini = _nouveau_software_context_fini,
         },
+        .vblank = nvc0_software_vblsem_release,
 };
 
 /*******************************************************************************
  * software engine/subdev functions
  ******************************************************************************/
 
-static int
-nvc0_software_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-                   struct nouveau_oclass *oclass, void *data, u32 size,
-                   struct nouveau_object **pobject)
-{
-        struct nvc0_software_priv *priv;
-        int ret;
-
-        ret = nouveau_software_create(parent, engine, oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        nv_engine(priv)->cclass = &nvc0_software_cclass;
-        nv_engine(priv)->sclass = nvc0_software_sclass;
-        nv_subdev(priv)->intr = nv04_software_intr;
-        return 0;
-}
-
-struct nouveau_oclass
-nvc0_software_oclass = {
-        .handle = NV_ENGINE(SW, 0xc0),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nvc0_software_ctor,
+struct nouveau_oclass *
+nvc0_software_oclass = &(struct nv50_software_oclass) {
+        .base.handle = NV_ENGINE(SW, 0xc0),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv50_software_ctor,
                 .dtor = _nouveau_software_dtor,
                 .init = _nouveau_software_init,
                 .fini = _nouveau_software_fini,
         },
-};
+        .cclass = &nvc0_software_cclass.base,
+        .sclass =  nvc0_software_sclass,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/include/core/class.h b/drivers/gpu/drm/nouveau/core/include/core/class.h
index 5a5961b6a6a3..560c3593dae7 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/class.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/class.h
@@ -22,7 +22,7 @@
 #define NV_DEVICE_DISABLE_PPP                             0x0000004000000000ULL
 #define NV_DEVICE_DISABLE_COPY0                           0x0000008000000000ULL
 #define NV_DEVICE_DISABLE_COPY1                           0x0000010000000000ULL
-#define NV_DEVICE_DISABLE_UNK1C1                          0x0000020000000000ULL
+#define NV_DEVICE_DISABLE_VIC                             0x0000020000000000ULL
 #define NV_DEVICE_DISABLE_VENC                            0x0000040000000000ULL
 
 struct nv_device_class {
@@ -98,6 +98,77 @@ struct nv_dma_class {
         u32 conf0;
 };
 
+/* Perfmon counter class
+ *
+ * XXXX: NV_PERFCTR
+ */
+#define NV_PERFCTR_CLASS                                             0x0000ffff
+#define NV_PERFCTR_QUERY                                             0x00000000
+#define NV_PERFCTR_SAMPLE                                            0x00000001
+#define NV_PERFCTR_READ                                              0x00000002
+
+struct nv_perfctr_class {
+        u16 logic_op;
+        struct {
+                char __user *name; /*XXX: use cfu when exposed to userspace */
+                u32 size;
+        } signal[4];
+};
+
+struct nv_perfctr_query {
+        u32 iter;
+        u32 size;
+        char __user *name; /*XXX: use ctu when exposed to userspace */
+};
+
+struct nv_perfctr_sample {
+};
+
+struct nv_perfctr_read {
+        u32 ctr;
+        u32 clk;
+};
+
+/* Device control class
+ *
+ * XXXX: NV_CONTROL
+ */
+#define NV_CONTROL_CLASS                                             0x0000fffe
+
+#define NV_CONTROL_PSTATE_INFO                                       0x00000000
+#define NV_CONTROL_PSTATE_INFO_USTATE_DISABLE                              (-1)
+#define NV_CONTROL_PSTATE_INFO_USTATE_PERFMON                              (-2)
+#define NV_CONTROL_PSTATE_INFO_PSTATE_UNKNOWN                              (-1)
+#define NV_CONTROL_PSTATE_INFO_PSTATE_PERFMON                              (-2)
+#define NV_CONTROL_PSTATE_ATTR                                       0x00000001
+#define NV_CONTROL_PSTATE_ATTR_STATE_CURRENT                               (-1)
+#define NV_CONTROL_PSTATE_USER                                       0x00000002
+#define NV_CONTROL_PSTATE_USER_STATE_UNKNOWN                               (-1)
+#define NV_CONTROL_PSTATE_USER_STATE_PERFMON                               (-2)
+
+struct nv_control_pstate_info {
+        u32 count; /* out: number of power states */
+        s32 ustate; /* out: current target pstate index */
+        u32 pstate; /* out: current pstate index */
+};
+
+struct nv_control_pstate_attr {
+        s32 state; /*  in: index of pstate to query
+                    * out: pstate identifier
+                    */
+        u32 index; /*  in: index of attribute to query
+                    * out: index of next attribute, or 0 if no more
+                    */
+        char name[32];
+        char unit[16];
+        u32 min;
+        u32 max;
+};
+
+struct nv_control_pstate_user {
+        s32 state; /*  in: pstate identifier */
+};
+
 /* DMA FIFO channel classes
  *
  * 006b: NV03_CHANNEL_DMA
diff --git a/drivers/gpu/drm/nouveau/core/include/core/debug.h b/drivers/gpu/drm/nouveau/core/include/core/debug.h
index 9ea18dfcb4d0..8092e2e90323 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/debug.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/debug.h
@@ -1,13 +1,20 @@
 #ifndef __NOUVEAU_DEBUG_H__
 #define __NOUVEAU_DEBUG_H__
 
+extern int nv_info_debug_level;
+
 #define NV_DBG_FATAL    0
 #define NV_DBG_ERROR    1
 #define NV_DBG_WARN     2
-#define NV_DBG_INFO     3
+#define NV_DBG_INFO     nv_info_debug_level
 #define NV_DBG_DEBUG    4
 #define NV_DBG_TRACE    5
 #define NV_DBG_PARANOIA 6
 #define NV_DBG_SPAM     7
 
+#define NV_DBG_INFO_NORMAL 3
+#define NV_DBG_INFO_SILENT NV_DBG_DEBUG
+
+#define nv_debug_level(a) nv_info_debug_level = NV_DBG_INFO_##a
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/device.h b/drivers/gpu/drm/nouveau/core/include/core/device.h
index 99b6600fe80a..ac2881d1776a 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/device.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/device.h
@@ -33,9 +33,10 @@ enum nv_subdev_type {
         NVDEV_SUBDEV_INSTMEM,
         NVDEV_SUBDEV_VM,
         NVDEV_SUBDEV_BAR,
+        NVDEV_SUBDEV_PWR,
         NVDEV_SUBDEV_VOLT,
-        NVDEV_SUBDEV_CLOCK,
         NVDEV_SUBDEV_THERM,
+        NVDEV_SUBDEV_CLOCK,
 
         NVDEV_ENGINE_DMAOBJ,
         NVDEV_ENGINE_FIFO,
@@ -50,9 +51,10 @@ enum nv_subdev_type {
         NVDEV_ENGINE_COPY0,
         NVDEV_ENGINE_COPY1,
         NVDEV_ENGINE_COPY2,
-        NVDEV_ENGINE_UNK1C1,
+        NVDEV_ENGINE_VIC,
         NVDEV_ENGINE_VENC,
         NVDEV_ENGINE_DISP,
+        NVDEV_ENGINE_PERFMON,
 
         NVDEV_SUBDEV_NR,
 };
@@ -72,6 +74,7 @@ struct nouveau_device {
         enum {
                 NV_04    = 0x04,
                 NV_10    = 0x10,
+                NV_11    = 0x11,
                 NV_20    = 0x20,
                 NV_30    = 0x30,
                 NV_40    = 0x40,
diff --git a/drivers/gpu/drm/nouveau/core/include/core/event.h b/drivers/gpu/drm/nouveau/core/include/core/event.h
index 9e094408f14e..5d539ebff3ed 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/event.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/event.h
@@ -5,13 +5,21 @@
 #define NVKM_EVENT_DROP 0
 #define NVKM_EVENT_KEEP 1
 
+/* nouveau_eventh.flags bit #s */
+#define NVKM_EVENT_ENABLE 0
+
 struct nouveau_eventh {
+        struct nouveau_event *event;
         struct list_head head;
-        int (*func)(struct nouveau_eventh *, int index);
+        unsigned long flags;
+        int index;
+        int (*func)(void *, int);
+        void *priv;
 };
 
 struct nouveau_event {
-        spinlock_t lock;
+        spinlock_t list_lock;
+        spinlock_t refs_lock;
 
         void *priv;
         void (*enable)(struct nouveau_event *, int index);
@@ -28,9 +36,11 @@ int  nouveau_event_create(int index_nr, struct nouveau_event **);
 void nouveau_event_destroy(struct nouveau_event **);
 void nouveau_event_trigger(struct nouveau_event *, int index);
 
-void nouveau_event_get(struct nouveau_event *, int index,
-                       struct nouveau_eventh *);
-void nouveau_event_put(struct nouveau_event *, int index,
-                       struct nouveau_eventh *);
+int  nouveau_event_new(struct nouveau_event *, int index,
+                       int (*func)(void *, int), void *,
+                       struct nouveau_eventh **);
+void nouveau_event_ref(struct nouveau_eventh *, struct nouveau_eventh **);
+void nouveau_event_get(struct nouveau_eventh *);
+void nouveau_event_put(struct nouveau_eventh *);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/option.h b/drivers/gpu/drm/nouveau/core/include/core/option.h
index 27074957fd21..ed055847887e 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/option.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/option.h
@@ -8,4 +8,13 @@ bool nouveau_boolopt(const char *optstr, const char *opt, bool value);
 
 int nouveau_dbgopt(const char *optstr, const char *sub);
 
+/* compares unterminated string 'str' with zero-terminated string 'cmp' */
+static inline int
+strncasecmpz(const char *str, const char *cmp, size_t len)
+{
+        if (strlen(cmp) != len)
+                return len;
+        return strncasecmp(str, cmp, len);
+}
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/printk.h b/drivers/gpu/drm/nouveau/core/include/core/printk.h
index d87836e3a704..0f9a37bd32b0 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/printk.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/printk.h
@@ -6,27 +6,12 @@
 
 struct nouveau_object;
 
-#define NV_PRINTK_FATAL    KERN_CRIT
-#define NV_PRINTK_ERROR    KERN_ERR
-#define NV_PRINTK_WARN     KERN_WARNING
-#define NV_PRINTK_INFO     KERN_INFO
-#define NV_PRINTK_DEBUG    KERN_DEBUG
-#define NV_PRINTK_PARANOIA KERN_DEBUG
-#define NV_PRINTK_TRACE    KERN_DEBUG
-#define NV_PRINTK_SPAM     KERN_DEBUG
-
-extern int nv_printk_suspend_level;
-
-#define NV_DBG_SUSPEND (nv_printk_suspend_level)
-#define NV_PRINTK_SUSPEND  (nv_printk_level_to_pfx(nv_printk_suspend_level))
-
-const char *nv_printk_level_to_pfx(int level);
-void __printf(4, 5)
-nv_printk_(struct nouveau_object *, const char *, int, const char *, ...);
+void __printf(3, 4)
+nv_printk_(struct nouveau_object *, int, const char *, ...);
 
 #define nv_printk(o,l,f,a...) do {                                             \
         if (NV_DBG_##l <= CONFIG_NOUVEAU_DEBUG)                                \
-                nv_printk_(nv_object(o), NV_PRINTK_##l, NV_DBG_##l, f, ##a);   \
+                nv_printk_(nv_object(o), NV_DBG_##l, f, ##a);                  \
 } while(0)
 
 #define nv_fatal(o,f,a...) nv_printk((o), FATAL, f, ##a)
@@ -37,16 +22,9 @@ nv_printk_(struct nouveau_object *, const char *, int, const char *, ...);
 #define nv_trace(o,f,a...) nv_printk((o), TRACE, f, ##a)
 #define nv_spam(o,f,a...) nv_printk((o), SPAM, f, ##a)
 
-#define nv_suspend(o,f,a...) nv_printk((o), SUSPEND, f, ##a)
-
-static inline void nv_suspend_set_printk_level(int level)
-{
-        nv_printk_suspend_level = level;
-}
-
 #define nv_assert(f,a...) do {                                                 \
         if (NV_DBG_FATAL <= CONFIG_NOUVEAU_DEBUG)                              \
-                nv_printk_(NULL, NV_PRINTK_FATAL, NV_DBG_FATAL, f "\n", ##a);  \
+                nv_printk_(NULL, NV_DBG_FATAL, f "\n", ##a);                   \
         BUG_ON(1);                                                             \
 } while(0)
 
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/fifo.h b/drivers/gpu/drm/nouveau/core/include/engine/fifo.h
index 633c2f806482..8c32cf4d83c7 100644
--- a/drivers/gpu/drm/nouveau/core/include/engine/fifo.h
+++ b/drivers/gpu/drm/nouveau/core/include/engine/fifo.h
@@ -101,14 +101,14 @@ nouveau_client_name_for_fifo_chid(struct nouveau_fifo *fifo, u32 chid);
 #define _nouveau_fifo_init _nouveau_engine_init
 #define _nouveau_fifo_fini _nouveau_engine_fini
 
-extern struct nouveau_oclass nv04_fifo_oclass;
-extern struct nouveau_oclass nv10_fifo_oclass;
-extern struct nouveau_oclass nv17_fifo_oclass;
-extern struct nouveau_oclass nv40_fifo_oclass;
-extern struct nouveau_oclass nv50_fifo_oclass;
-extern struct nouveau_oclass nv84_fifo_oclass;
-extern struct nouveau_oclass nvc0_fifo_oclass;
-extern struct nouveau_oclass nve0_fifo_oclass;
+extern struct nouveau_oclass *nv04_fifo_oclass;
+extern struct nouveau_oclass *nv10_fifo_oclass;
+extern struct nouveau_oclass *nv17_fifo_oclass;
+extern struct nouveau_oclass *nv40_fifo_oclass;
+extern struct nouveau_oclass *nv50_fifo_oclass;
+extern struct nouveau_oclass *nv84_fifo_oclass;
+extern struct nouveau_oclass *nvc0_fifo_oclass;
+extern struct nouveau_oclass *nve0_fifo_oclass;
 
 void nv04_fifo_intr(struct nouveau_subdev *);
 int  nv04_fifo_context_attach(struct nouveau_object *, struct nouveau_object *);
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/mpeg.h b/drivers/gpu/drm/nouveau/core/include/engine/mpeg.h
index 1d1a89a06ee4..9b0d938199f6 100644
--- a/drivers/gpu/drm/nouveau/core/include/engine/mpeg.h
+++ b/drivers/gpu/drm/nouveau/core/include/engine/mpeg.h
@@ -42,10 +42,13 @@ struct nouveau_mpeg {
 
 extern struct nouveau_oclass nv31_mpeg_oclass;
 extern struct nouveau_oclass nv40_mpeg_oclass;
+extern struct nouveau_oclass nv44_mpeg_oclass;
 extern struct nouveau_oclass nv50_mpeg_oclass;
 extern struct nouveau_oclass nv84_mpeg_oclass;
-
+extern struct nouveau_ofuncs nv31_mpeg_ofuncs;
+extern struct nouveau_oclass nv31_mpeg_cclass;
 extern struct nouveau_oclass nv31_mpeg_sclass[];
+extern struct nouveau_oclass nv40_mpeg_sclass[];
 void nv31_mpeg_intr(struct nouveau_subdev *);
 void nv31_mpeg_tile_prog(struct nouveau_engine *, int);
 int  nv31_mpeg_init(struct nouveau_object *);
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/perfmon.h b/drivers/gpu/drm/nouveau/core/include/engine/perfmon.h
new file mode 100644
index 000000000000..49b0024910fe
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/engine/perfmon.h
@@ -0,0 +1,39 @@
+#ifndef __NVKM_PERFMON_H__
+#define __NVKM_PERFMON_H__
+
+#include <core/device.h>
+#include <core/engine.h>
+#include <core/engctx.h>
+#include <core/class.h>
+
+struct nouveau_perfdom;
+struct nouveau_perfctr;
+struct nouveau_perfmon {
+        struct nouveau_engine base;
+
+        struct nouveau_perfctx *context;
+        void *profile_data;
+
+        struct list_head domains;
+        u32 sequence;
+
+        /*XXX: temp for daemon backend */
+        u32 pwr[8];
+        u32 last;
+};
+
+static inline struct nouveau_perfmon *
+nouveau_perfmon(void *obj)
+{
+        return (void *)nv_device(obj)->subdev[NVDEV_ENGINE_PERFMON];
+}
+
+extern struct nouveau_oclass *nv40_perfmon_oclass;
+extern struct nouveau_oclass *nv50_perfmon_oclass;
+extern struct nouveau_oclass *nv84_perfmon_oclass;
+extern struct nouveau_oclass *nva3_perfmon_oclass;
+extern struct nouveau_oclass nvc0_perfmon_oclass;
+extern struct nouveau_oclass nve0_perfmon_oclass;
+extern struct nouveau_oclass nvf0_perfmon_oclass;
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/software.h b/drivers/gpu/drm/nouveau/core/include/engine/software.h
index 45799487e573..23a462b50d03 100644
--- a/drivers/gpu/drm/nouveau/core/include/engine/software.h
+++ b/drivers/gpu/drm/nouveau/core/include/engine/software.h
@@ -3,19 +3,10 @@
 
 #include <core/engine.h>
 #include <core/engctx.h>
-#include <core/event.h>
 
 struct nouveau_software_chan {
         struct nouveau_engctx base;
 
-        struct {
-                struct nouveau_eventh event;
-                u32 channel;
-                u32 ctxdma;
-                u64 offset;
-                u32 value;
-        } vblank;
-
         int (*flip)(void *);
         void *flip_data;
 };
@@ -50,10 +41,10 @@ struct nouveau_software {
 #define _nouveau_software_init _nouveau_engine_init
 #define _nouveau_software_fini _nouveau_engine_fini
 
-extern struct nouveau_oclass nv04_software_oclass;
-extern struct nouveau_oclass nv10_software_oclass;
-extern struct nouveau_oclass nv50_software_oclass;
-extern struct nouveau_oclass nvc0_software_oclass;
+extern struct nouveau_oclass *nv04_software_oclass;
+extern struct nouveau_oclass *nv10_software_oclass;
+extern struct nouveau_oclass *nv50_software_oclass;
+extern struct nouveau_oclass *nvc0_software_oclass;
 
 void nv04_software_intr(struct nouveau_subdev *);
 
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/boost.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/boost.h
new file mode 100644
index 000000000000..662b20726851
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/boost.h
@@ -0,0 +1,29 @@
+#ifndef __NVBIOS_BOOST_H__
+#define __NVBIOS_BOOST_H__
+
+u16 nvbios_boostTe(struct nouveau_bios *, u8 *, u8 *, u8 *, u8 *, u8 *, u8 *);
+
+struct nvbios_boostE {
+        u8  pstate;
+        u32 min;
+        u32 max;
+};
+
+u16 nvbios_boostEe(struct nouveau_bios *, int idx, u8 *, u8 *, u8 *, u8 *);
+u16 nvbios_boostEp(struct nouveau_bios *, int idx, u8 *, u8 *, u8 *, u8 *,
+                   struct nvbios_boostE *);
+u16 nvbios_boostEm(struct nouveau_bios *, u8, u8 *, u8 *, u8 *, u8 *,
+                   struct nvbios_boostE *);
+
+struct nvbios_boostS {
+        u8  domain;
+        u8  percent;
+        u32 min;
+        u32 max;
+};
+
+u16 nvbios_boostSe(struct nouveau_bios *, int, u16, u8 *, u8 *, u8, u8);
+u16 nvbios_boostSp(struct nouveau_bios *, int, u16, u8 *, u8 *, u8, u8,
+                   struct nvbios_boostS *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/cstep.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/cstep.h
new file mode 100644
index 000000000000..a80a43809883
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/cstep.h
@@ -0,0 +1,28 @@
+#ifndef __NVBIOS_CSTEP_H__
+#define __NVBIOS_CSTEP_H__
+
+u16 nvbios_cstepTe(struct nouveau_bios *,
+                   u8 *ver, u8 *hdr, u8 *cnt, u8 *len, u8 *xnr, u8 *xsz);
+
+struct nvbios_cstepE {
+        u8  pstate;
+        u8  index;
+};
+
+u16 nvbios_cstepEe(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr);
+u16 nvbios_cstepEp(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr,
+                   struct nvbios_cstepE *);
+u16 nvbios_cstepEm(struct nouveau_bios *, u8 pstate, u8 *ver, u8 *hdr,
+                   struct nvbios_cstepE *);
+
+struct nvbios_cstepX {
+        u32 freq;
+        u8  unkn[2];
+        u8  voltage;
+};
+
+u16 nvbios_cstepXe(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr);
+u16 nvbios_cstepXp(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr,
+                   struct nvbios_cstepX *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/gpio.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/gpio.h
index 96d3364f6db3..c7b2e586be0b 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/bios/gpio.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/gpio.h
@@ -7,7 +7,15 @@ enum dcb_gpio_func_name {
         DCB_GPIO_TVDAC1 = 0x2d,
         DCB_GPIO_FAN = 0x09,
         DCB_GPIO_FAN_SENSE = 0x3d,
-        DCB_GPIO_UNUSED = 0xff
+        DCB_GPIO_UNUSED = 0xff,
+        DCB_GPIO_VID0 = 0x04,
+        DCB_GPIO_VID1 = 0x05,
+        DCB_GPIO_VID2 = 0x06,
+        DCB_GPIO_VID3 = 0x1a,
+        DCB_GPIO_VID4 = 0x73,
+        DCB_GPIO_VID5 = 0x74,
+        DCB_GPIO_VID6 = 0x75,
+        DCB_GPIO_VID7 = 0x76,
 };
 
 #define DCB_GPIO_LOG_DIR     0x02
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/perf.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/perf.h
index 0b285e99be5a..16ff06ec2a88 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/bios/perf.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/perf.h
@@ -3,6 +3,39 @@
 
 struct nouveau_bios;
 
+u16 nvbios_perf_table(struct nouveau_bios *, u8 *ver, u8 *hdr,
+                      u8 *cnt, u8 *len, u8 *snr, u8 *ssz);
+
+struct nvbios_perfE {
+        u8  pstate;
+        u8  fanspeed;
+        u8  voltage;
+        u32 core;
+        u32 shader;
+        u32 memory;
+        u32 vdec;
+        u32 disp;
+        u32 script;
+};
+
+u16 nvbios_perf_entry(struct nouveau_bios *, int idx,
+                      u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+u16 nvbios_perfEp(struct nouveau_bios *, int idx,
+                  u8 *ver, u8 *hdr, u8 *cnt, u8 *len, struct nvbios_perfE *);
+
+struct nvbios_perfS {
+        union {
+                struct {
+                        u32 freq;
+                } v40;
+        };
+};
+
+u32 nvbios_perfSe(struct nouveau_bios *, u32 data, int idx,
+                  u8 *ver, u8 *hdr, u8 cnt, u8 len);
+u32 nvbios_perfSp(struct nouveau_bios *, u32 data, int idx,
+                  u8 *ver, u8 *hdr, u8 cnt, u8 len, struct nvbios_perfS *);
+
 struct nvbios_perf_fan {
         u32 pwm_divisor;
 };
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/rammap.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/rammap.h
new file mode 100644
index 000000000000..bc15e0320877
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/rammap.h
@@ -0,0 +1,11 @@
+#ifndef __NVBIOS_RAMMAP_H__
+#define __NVBIOS_RAMMAP_H__
+
+u16 nvbios_rammap_table(struct nouveau_bios *, u8 *ver, u8 *hdr,
+                        u8 *cnt, u8 *len, u8 *snr, u8 *ssz);
+u16 nvbios_rammap_entry(struct nouveau_bios *, int idx,
+                        u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+u16 nvbios_rammap_match(struct nouveau_bios *, u16 khz,
+                        u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/timing.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/timing.h
new file mode 100644
index 000000000000..963694b54224
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/timing.h
@@ -0,0 +1,8 @@
+#ifndef __NVBIOS_TIMING_H__
+#define __NVBIOS_TIMING_H__
+
+u16 nvbios_timing_table(struct nouveau_bios *,
+                        u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+u16 nvbios_timing_entry(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/vmap.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/vmap.h
new file mode 100644
index 000000000000..ad5a8f20e113
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/vmap.h
@@ -0,0 +1,25 @@
+#ifndef __NVBIOS_VMAP_H__
+#define __NVBIOS_VMAP_H__
+
+struct nouveau_bios;
+
+struct nvbios_vmap {
+};
+
+u16 nvbios_vmap_table(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+u16 nvbios_vmap_parse(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+                      struct nvbios_vmap *);
+
+struct nvbios_vmap_entry {
+        u8  unk0;
+        u8  link;
+        u32 min;
+        u32 max;
+        s32 arg[6];
+};
+
+u16 nvbios_vmap_entry(struct nouveau_bios *, int idx, u8 *ver, u8 *len);
+u16 nvbios_vmap_entry_parse(struct nouveau_bios *, int idx, u8 *ver, u8 *len,
+                            struct nvbios_vmap_entry *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/volt.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/volt.h
new file mode 100644
index 000000000000..6a11dcd59770
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/volt.h
@@ -0,0 +1,27 @@
+#ifndef __NVBIOS_VOLT_H__
+#define __NVBIOS_VOLT_H__
+
+struct nouveau_bios;
+
+struct nvbios_volt {
+        u8  vidmask;
+        u32 min;
+        u32 max;
+        u32 base;
+        s16 step;
+};
+
+u16 nvbios_volt_table(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len);
+u16 nvbios_volt_parse(struct nouveau_bios *, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+                      struct nvbios_volt *);
+
+struct nvbios_volt_entry {
+        u32 voltage;
+        u8  vid;
+};
+
+u16 nvbios_volt_entry(struct nouveau_bios *, int idx, u8 *ver, u8 *len);
+u16 nvbios_volt_entry_parse(struct nouveau_bios *, int idx, u8 *ver, u8 *len,
+                            struct nvbios_volt_entry *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bus.h b/drivers/gpu/drm/nouveau/core/include/subdev/bus.h
index 7d88ec4a6d06..697f7ce70aab 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/bus.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bus.h
@@ -11,6 +11,8 @@ struct nouveau_bus_intr {
 
 struct nouveau_bus {
         struct nouveau_subdev base;
+        int (*hwsq_exec)(struct nouveau_bus *, u32 *, u32);
+        u32 hwsq_size;
 };
 
 static inline struct nouveau_bus *
@@ -33,9 +35,19 @@ nouveau_bus(void *obj)
 #define _nouveau_bus_init _nouveau_subdev_init
 #define _nouveau_bus_fini _nouveau_subdev_fini
 
-extern struct nouveau_oclass nv04_bus_oclass;
-extern struct nouveau_oclass nv31_bus_oclass;
-extern struct nouveau_oclass nv50_bus_oclass;
-extern struct nouveau_oclass nvc0_bus_oclass;
+extern struct nouveau_oclass *nv04_bus_oclass;
+extern struct nouveau_oclass *nv31_bus_oclass;
+extern struct nouveau_oclass *nv50_bus_oclass;
+extern struct nouveau_oclass *nv94_bus_oclass;
+extern struct nouveau_oclass *nvc0_bus_oclass;
+
+/* interface to sequencer */
+struct nouveau_hwsq;
+int  nouveau_hwsq_init(struct nouveau_bus *, struct nouveau_hwsq **);
+int  nouveau_hwsq_fini(struct nouveau_hwsq **, bool exec);
+void nouveau_hwsq_wr32(struct nouveau_hwsq *, u32 addr, u32 data);
+void nouveau_hwsq_setf(struct nouveau_hwsq *, u8 flag, int data);
+void nouveau_hwsq_wait(struct nouveau_hwsq *, u8 flag, u8 data);
+void nouveau_hwsq_nsec(struct nouveau_hwsq *, u32 nsec);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/clock.h b/drivers/gpu/drm/nouveau/core/include/subdev/clock.h
index 89ee289097a6..e2675bc0edba 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/clock.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/clock.h
@@ -7,9 +7,78 @@
 struct nouveau_pll_vals;
 struct nvbios_pll;
 
+enum nv_clk_src {
+        nv_clk_src_crystal,
+        nv_clk_src_href,
+
+        nv_clk_src_hclk,
+        nv_clk_src_hclkm3,
+        nv_clk_src_hclkm3d2,
+
+        nv_clk_src_host,
+
+        nv_clk_src_sppll0,
+        nv_clk_src_sppll1,
+
+        nv_clk_src_mpllsrcref,
+        nv_clk_src_mpllsrc,
+        nv_clk_src_mpll,
+        nv_clk_src_mdiv,
+
+        nv_clk_src_core,
+        nv_clk_src_shader,
+
+        nv_clk_src_mem,
+
+        nv_clk_src_gpc,
+        nv_clk_src_rop,
+        nv_clk_src_hubk01,
+        nv_clk_src_hubk06,
+        nv_clk_src_hubk07,
+        nv_clk_src_copy,
+        nv_clk_src_daemon,
+        nv_clk_src_disp,
+        nv_clk_src_vdec,
+
+        nv_clk_src_dom6,
+
+        nv_clk_src_max,
+};
+
+struct nouveau_cstate {
+        struct list_head head;
+        u8  voltage;
+        u32 domain[nv_clk_src_max];
+};
+
+struct nouveau_pstate {
+        struct list_head head;
+        struct list_head list; /* c-states */
+        struct nouveau_cstate base;
+        u8 pstate;
+        u8 fanspeed;
+};
+
 struct nouveau_clock {
         struct nouveau_subdev base;
 
+        struct nouveau_clocks *domains;
+        struct nouveau_pstate bstate;
+
+        struct list_head states;
+        int state_nr;
+
+        int pstate; /* current */
+        int ustate; /* user-requested (-1 disabled, -2 perfmon) */
+        int astate; /* perfmon adjustment (base) */
+        int tstate; /* thermal adjustment (max-) */
+        int dstate; /* display adjustment (min+) */
+
+        int  (*read)(struct nouveau_clock *, enum nv_clk_src);
+        int  (*calc)(struct nouveau_clock *, struct nouveau_cstate *);
+        int  (*prog)(struct nouveau_clock *);
+        void (*tidy)(struct nouveau_clock *);
+
         /*XXX: die, these are here *only* to support the completely
          *     bat-shit insane what-was-nouveau_hw.c code
          */
@@ -25,27 +94,42 @@ nouveau_clock(void *obj)
         return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_CLOCK];
 }
 
-#define nouveau_clock_create(p,e,o,d)                                          \
-        nouveau_subdev_create((p), (e), (o), 0, "CLOCK", "clock", d)
-#define nouveau_clock_destroy(p)                                               \
-        nouveau_subdev_destroy(&(p)->base)
-#define nouveau_clock_init(p)                                                  \
-        nouveau_subdev_init(&(p)->base)
+struct nouveau_clocks {
+        enum nv_clk_src name;
+        u8 bios; /* 0xff for none */
+#define NVKM_CLK_DOM_FLAG_CORE 0x01
+        u8 flags;
+        const char *mname;
+        int mdiv;
+};
+
+#define nouveau_clock_create(p,e,o,i,d)                                        \
+        nouveau_clock_create_((p), (e), (o), (i), sizeof(**d), (void **)d)
+#define nouveau_clock_destroy(p) ({                                            \
+        struct nouveau_clock *clk = (p);                                       \
+        _nouveau_clock_dtor(nv_object(clk));                                   \
+})
+#define nouveau_clock_init(p) ({                                               \
+        struct nouveau_clock *clk = (p);                                       \
+        _nouveau_clock_init(nv_object(clk));                                   \
+})
 #define nouveau_clock_fini(p,s)                                                \
         nouveau_subdev_fini(&(p)->base, (s))
 
 int  nouveau_clock_create_(struct nouveau_object *, struct nouveau_object *,
-                           struct nouveau_oclass *, void *, u32, int, void **);
-
-#define _nouveau_clock_dtor _nouveau_subdev_dtor
-#define _nouveau_clock_init _nouveau_subdev_init
+                           struct nouveau_oclass *,
+                           struct nouveau_clocks *, int, void **);
+void _nouveau_clock_dtor(struct nouveau_object *);
+int _nouveau_clock_init(struct nouveau_object *);
 #define _nouveau_clock_fini _nouveau_subdev_fini
 
 extern struct nouveau_oclass nv04_clock_oclass;
 extern struct nouveau_oclass nv40_clock_oclass;
-extern struct nouveau_oclass nv50_clock_oclass;
+extern struct nouveau_oclass *nv50_clock_oclass;
+extern struct nouveau_oclass *nv84_clock_oclass;
 extern struct nouveau_oclass nva3_clock_oclass;
 extern struct nouveau_oclass nvc0_clock_oclass;
+extern struct nouveau_oclass nve0_clock_oclass;
 
 int nv04_clock_pll_set(struct nouveau_clock *, u32 type, u32 freq);
 int nv04_clock_pll_calc(struct nouveau_clock *, struct nvbios_pll *,
@@ -55,4 +139,9 @@ int nv04_clock_pll_prog(struct nouveau_clock *, u32 reg1,
 int nva3_clock_pll_calc(struct nouveau_clock *, struct nvbios_pll *,
                         int clk, struct nouveau_pll_vals *);
 
+int nouveau_clock_ustate(struct nouveau_clock *, int req);
+int nouveau_clock_astate(struct nouveau_clock *, int req, int rel);
+int nouveau_clock_dstate(struct nouveau_clock *, int req, int rel);
+int nouveau_clock_tstate(struct nouveau_clock *, int req, int rel);
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/fb.h b/drivers/gpu/drm/nouveau/core/include/subdev/fb.h
index 2e7405084261..8541aa382ff2 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/fb.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/fb.h
@@ -78,23 +78,28 @@ nouveau_fb(void *obj)
         return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_FB];
 }
 
-extern struct nouveau_oclass nv04_fb_oclass;
-extern struct nouveau_oclass nv10_fb_oclass;
-extern struct nouveau_oclass nv1a_fb_oclass;
-extern struct nouveau_oclass nv20_fb_oclass;
-extern struct nouveau_oclass nv25_fb_oclass;
-extern struct nouveau_oclass nv30_fb_oclass;
-extern struct nouveau_oclass nv35_fb_oclass;
-extern struct nouveau_oclass nv36_fb_oclass;
-extern struct nouveau_oclass nv40_fb_oclass;
-extern struct nouveau_oclass nv41_fb_oclass;
-extern struct nouveau_oclass nv44_fb_oclass;
-extern struct nouveau_oclass nv46_fb_oclass;
-extern struct nouveau_oclass nv47_fb_oclass;
-extern struct nouveau_oclass nv49_fb_oclass;
-extern struct nouveau_oclass nv4e_fb_oclass;
-extern struct nouveau_oclass nv50_fb_oclass;
-extern struct nouveau_oclass nvc0_fb_oclass;
+extern struct nouveau_oclass *nv04_fb_oclass;
+extern struct nouveau_oclass *nv10_fb_oclass;
+extern struct nouveau_oclass *nv1a_fb_oclass;
+extern struct nouveau_oclass *nv20_fb_oclass;
+extern struct nouveau_oclass *nv25_fb_oclass;
+extern struct nouveau_oclass *nv30_fb_oclass;
+extern struct nouveau_oclass *nv35_fb_oclass;
+extern struct nouveau_oclass *nv36_fb_oclass;
+extern struct nouveau_oclass *nv40_fb_oclass;
+extern struct nouveau_oclass *nv41_fb_oclass;
+extern struct nouveau_oclass *nv44_fb_oclass;
+extern struct nouveau_oclass *nv46_fb_oclass;
+extern struct nouveau_oclass *nv47_fb_oclass;
+extern struct nouveau_oclass *nv49_fb_oclass;
+extern struct nouveau_oclass *nv4e_fb_oclass;
+extern struct nouveau_oclass *nv50_fb_oclass;
+extern struct nouveau_oclass *nv84_fb_oclass;
+extern struct nouveau_oclass *nva3_fb_oclass;
+extern struct nouveau_oclass *nvaa_fb_oclass;
+extern struct nouveau_oclass *nvaf_fb_oclass;
+extern struct nouveau_oclass *nvc0_fb_oclass;
+extern struct nouveau_oclass *nve0_fb_oclass;
 
 struct nouveau_ram {
         struct nouveau_object base;
@@ -121,6 +126,17 @@ struct nouveau_ram {
         int  (*get)(struct nouveau_fb *, u64 size, u32 align,
                     u32 size_nc, u32 type, struct nouveau_mem **);
         void (*put)(struct nouveau_fb *, struct nouveau_mem **);
+
+        int  (*calc)(struct nouveau_fb *, u32 freq);
+        int  (*prog)(struct nouveau_fb *);
+        void (*tidy)(struct nouveau_fb *);
+        struct {
+                u8  version;
+                u32 data;
+                u8  size;
+        } rammap, ramcfg, timing;
+        u32 freq;
+        u32 mr[16];
 };
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h b/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h
index 7e4e2775f249..9fa5da723871 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/i2c.h
@@ -60,13 +60,18 @@ void _nouveau_i2c_port_dtor(struct nouveau_object *);
 #define _nouveau_i2c_port_init nouveau_object_init
 #define _nouveau_i2c_port_fini nouveau_object_fini
 
+struct nouveau_i2c_board_info {
+        struct i2c_board_info dev;
+        u8 udelay; /* set to 0 to use the standard delay */
+};
+
 struct nouveau_i2c {
         struct nouveau_subdev base;
 
         struct nouveau_i2c_port *(*find)(struct nouveau_i2c *, u8 index);
         struct nouveau_i2c_port *(*find_type)(struct nouveau_i2c *, u16 type);
         int (*identify)(struct nouveau_i2c *, int index,
-                        const char *what, struct i2c_board_info *,
+                        const char *what, struct nouveau_i2c_board_info *,
                         bool (*match)(struct nouveau_i2c_port *,
                                       struct i2c_board_info *));
         struct list_head ports;
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/mc.h b/drivers/gpu/drm/nouveau/core/include/subdev/mc.h
index ce6569f365a7..adc88b73d911 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/mc.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/mc.h
@@ -11,7 +11,6 @@ struct nouveau_mc_intr {
 
 struct nouveau_mc {
         struct nouveau_subdev base;
-        const struct nouveau_mc_intr *intr_map;
         bool use_msi;
 };
 
@@ -21,8 +20,8 @@ nouveau_mc(void *obj)
         return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_MC];
 }
 
-#define nouveau_mc_create(p,e,o,m,d)                                           \
-        nouveau_mc_create_((p), (e), (o), (m), sizeof(**d), (void **)d)
+#define nouveau_mc_create(p,e,o,d)                                             \
+        nouveau_mc_create_((p), (e), (o), sizeof(**d), (void **)d)
 #define nouveau_mc_destroy(p) ({                                               \
         struct nouveau_mc *pmc = (p); _nouveau_mc_dtor(nv_object(pmc));        \
 })
@@ -34,20 +33,24 @@ nouveau_mc(void *obj)
 })
 
 int  nouveau_mc_create_(struct nouveau_object *, struct nouveau_object *,
-                        struct nouveau_oclass *, const struct nouveau_mc_intr *,
-                        int, void **);
+                        struct nouveau_oclass *, int, void **);
 void _nouveau_mc_dtor(struct nouveau_object *);
 int  _nouveau_mc_init(struct nouveau_object *);
 int  _nouveau_mc_fini(struct nouveau_object *, bool);
 
-extern struct nouveau_oclass nv04_mc_oclass;
-extern struct nouveau_oclass nv44_mc_oclass;
-extern struct nouveau_oclass nv50_mc_oclass;
-extern struct nouveau_oclass nv98_mc_oclass;
-extern struct nouveau_oclass nvc0_mc_oclass;
+struct nouveau_mc_oclass {
+        struct nouveau_oclass base;
+        const struct nouveau_mc_intr *intr;
+        void (*msi_rearm)(struct nouveau_mc *);
+};
 
-extern const struct nouveau_mc_intr nv04_mc_intr[];
-int nv04_mc_init(struct nouveau_object *);
-int nv50_mc_init(struct nouveau_object *);
+extern struct nouveau_oclass *nv04_mc_oclass;
+extern struct nouveau_oclass *nv40_mc_oclass;
+extern struct nouveau_oclass *nv44_mc_oclass;
+extern struct nouveau_oclass *nv50_mc_oclass;
+extern struct nouveau_oclass *nv94_mc_oclass;
+extern struct nouveau_oclass *nv98_mc_oclass;
+extern struct nouveau_oclass *nvc0_mc_oclass;
+extern struct nouveau_oclass *nvc3_mc_oclass;
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h b/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h
new file mode 100644
index 000000000000..c5c92cbed33f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/pwr.h
@@ -0,0 +1,80 @@
+#ifndef __NOUVEAU_PWR_H__
+#define __NOUVEAU_PWR_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+
+struct nouveau_pwr {
+        struct nouveau_subdev base;
+
+        struct {
+                u32 limit;
+                u32 *data;
+                u32  size;
+        } code;
+
+        struct {
+                u32 limit;
+                u32 *data;
+                u32  size;
+        } data;
+
+        struct {
+                u32 base;
+                u32 size;
+        } send;
+
+        struct {
+                u32 base;
+                u32 size;
+
+                struct work_struct work;
+                wait_queue_head_t wait;
+                u32 process;
+                u32 message;
+                u32 data[2];
+        } recv;
+
+        int (*message)(struct nouveau_pwr *, u32[2], u32, u32, u32, u32);
+};
+
+static inline struct nouveau_pwr *
+nouveau_pwr(void *obj)
+{
+        return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_PWR];
+}
+
+#define nouveau_pwr_create(p, e, o, d)                                         \
+        nouveau_pwr_create_((p), (e), (o), sizeof(**d), (void **)d)
+#define nouveau_pwr_destroy(p)                                                 \
+        nouveau_subdev_destroy(&(p)->base)
+#define nouveau_pwr_init(p) ({                                                 \
+        struct nouveau_pwr *ppwr = (p);                                        \
+        _nouveau_pwr_init(nv_object(ppwr));                                    \
+})
+#define nouveau_pwr_fini(p,s) ({                                               \
+        struct nouveau_pwr *ppwr = (p);                                        \
+        _nouveau_pwr_fini(nv_object(ppwr), (s));                               \
+})
+
+int nouveau_pwr_create_(struct nouveau_object *, struct nouveau_object *,
+                           struct nouveau_oclass *, int, void **);
+#define _nouveau_pwr_dtor _nouveau_subdev_dtor
+int _nouveau_pwr_init(struct nouveau_object *);
+int _nouveau_pwr_fini(struct nouveau_object *, bool);
+
+extern struct nouveau_oclass nva3_pwr_oclass;
+extern struct nouveau_oclass nvc0_pwr_oclass;
+extern struct nouveau_oclass nvd0_pwr_oclass;
+extern struct nouveau_oclass nv108_pwr_oclass;
+
+/* interface to MEMX process running on PPWR */
+struct nouveau_memx;
+int  nouveau_memx_init(struct nouveau_pwr *, struct nouveau_memx **);
+int  nouveau_memx_fini(struct nouveau_memx **, bool exec);
+void nouveau_memx_wr32(struct nouveau_memx *, u32 addr, u32 data);
+void nouveau_memx_wait(struct nouveau_memx *,
+                       u32 addr, u32 mask, u32 data, u32 nsec);
+void nouveau_memx_nsec(struct nouveau_memx *, u32 nsec);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/therm.h b/drivers/gpu/drm/nouveau/core/include/subdev/therm.h
index c075998d82e6..69891d4a3fe7 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/therm.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/therm.h
@@ -71,6 +71,8 @@ void _nouveau_therm_dtor(struct nouveau_object *);
 int  _nouveau_therm_init(struct nouveau_object *);
 int  _nouveau_therm_fini(struct nouveau_object *, bool);
 
+int  nouveau_therm_cstate(struct nouveau_therm *, int, int);
+
 extern struct nouveau_oclass nv40_therm_oclass;
 extern struct nouveau_oclass nv50_therm_oclass;
 extern struct nouveau_oclass nv84_therm_oclass;
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/volt.h b/drivers/gpu/drm/nouveau/core/include/subdev/volt.h
new file mode 100644
index 000000000000..820b62ffd75b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/volt.h
@@ -0,0 +1,60 @@
+#ifndef __NOUVEAU_VOLT_H__
+#define __NOUVEAU_VOLT_H__
+
+#include <core/subdev.h>
+#include <core/device.h>
+
+struct nouveau_voltage {
+        u32 uv;
+        u8  id;
+};
+
+struct nouveau_volt {
+        struct nouveau_subdev base;
+
+        int (*vid_get)(struct nouveau_volt *);
+        int (*get)(struct nouveau_volt *);
+        int (*vid_set)(struct nouveau_volt *, u8 vid);
+        int (*set)(struct nouveau_volt *, u32 uv);
+        int (*set_id)(struct nouveau_volt *, u8 id, int condition);
+
+        u8 vid_mask;
+        u8 vid_nr;
+        struct {
+                u32 uv;
+                u8 vid;
+        } vid[256];
+};
+
+static inline struct nouveau_volt *
+nouveau_volt(void *obj)
+{
+        return (void *)nv_device(obj)->subdev[NVDEV_SUBDEV_VOLT];
+}
+
+#define nouveau_volt_create(p, e, o, d)                                        \
+        nouveau_volt_create_((p), (e), (o), sizeof(**d), (void **)d)
+#define nouveau_volt_destroy(p) ({                                             \
+        struct nouveau_volt *v = (p);                                          \
+        _nouveau_volt_dtor(nv_object(v));                                      \
+})
+#define nouveau_volt_init(p) ({                                                \
+        struct nouveau_volt *v = (p);                                          \
+        _nouveau_volt_init(nv_object(v));                                      \
+})
+#define nouveau_volt_fini(p,s)                                                 \
+        nouveau_subdev_fini((p), (s))
+
+int  nouveau_volt_create_(struct nouveau_object *, struct nouveau_object *,
+                          struct nouveau_oclass *, int, void **);
+void _nouveau_volt_dtor(struct nouveau_object *);
+int  _nouveau_volt_init(struct nouveau_object *);
+#define _nouveau_volt_fini _nouveau_subdev_fini
+
+extern struct nouveau_oclass nv40_volt_oclass;
+
+int nouveau_voltgpio_init(struct nouveau_volt *);
+int nouveau_voltgpio_get(struct nouveau_volt *);
+int nouveau_voltgpio_set(struct nouveau_volt *, u8);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/boost.c b/drivers/gpu/drm/nouveau/core/subdev/bios/boost.c
new file mode 100644
index 000000000000..c1835e591c44
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/boost.c
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/boost.h>
+
+u16
+nvbios_boostTe(struct nouveau_bios *bios,
+               u8 *ver, u8 *hdr, u8 *cnt, u8 *len, u8 *snr, u8 *ssz)
+{
+        struct bit_entry bit_P;
+        u16 boost = 0x0000;
+
+        if (!bit_entry(bios, 'P', &bit_P)) {
+                if (bit_P.version == 2)
+                        boost = nv_ro16(bios, bit_P.offset + 0x30);
+
+                if (boost) {
+                        *ver = nv_ro08(bios, boost + 0);
+                        switch (*ver) {
+                        case 0x11:
+                                *hdr = nv_ro08(bios, boost + 1);
+                                *cnt = nv_ro08(bios, boost + 5);
+                                *len = nv_ro08(bios, boost + 2);
+                                *snr = nv_ro08(bios, boost + 4);
+                                *ssz = nv_ro08(bios, boost + 3);
+                                return boost;
+                        default:
+                                break;
+                        }
+                }
+        }
+
+        return 0x0000;
+}
+
+u16
+nvbios_boostEe(struct nouveau_bios *bios, int idx,
+               u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+        u8  snr, ssz;
+        u16 data = nvbios_boostTe(bios, ver, hdr, cnt, len, &snr, &ssz);
+        if (data && idx < *cnt) {
+                data = data + *hdr + (idx * (*len + (snr * ssz)));
+                *hdr = *len;
+                *cnt = snr;
+                *len = ssz;
+                return data;
+        }
+        return 0x0000;
+}
+
+u16
+nvbios_boostEp(struct nouveau_bios *bios, int idx,
+               u8 *ver, u8 *hdr, u8 *cnt, u8 *len, struct nvbios_boostE *info)
+{
+        u16 data = nvbios_boostEe(bios, idx, ver, hdr, cnt, len);
+        memset(info, 0x00, sizeof(*info));
+        if (data) {
+                info->pstate = (nv_ro16(bios, data + 0x00) & 0x01e0) >> 5;
+                info->min    =  nv_ro16(bios, data + 0x02) * 1000;
+                info->max    =  nv_ro16(bios, data + 0x04) * 1000;
+        }
+        return data;
+}
+
+u16
+nvbios_boostEm(struct nouveau_bios *bios, u8 pstate,
+               u8 *ver, u8 *hdr, u8 *cnt, u8 *len, struct nvbios_boostE *info)
+{
+        u32 data, idx = 0;
+        while ((data = nvbios_boostEp(bios, idx++, ver, hdr, cnt, len, info))) {
+                if (info->pstate == pstate)
+                        break;
+        }
+        return data;
+}
+
+u16
+nvbios_boostSe(struct nouveau_bios *bios, int idx,
+               u16 data, u8 *ver, u8 *hdr, u8 cnt, u8 len)
+{
+        if (data && idx < cnt) {
+                data = data + *hdr + (idx * len);
+                *hdr = len;
+                return data;
+        }
+        return 0x0000;
+}
+
+u16
+nvbios_boostSp(struct nouveau_bios *bios, int idx,
+               u16 data, u8 *ver, u8 *hdr, u8 cnt, u8 len,
+               struct nvbios_boostS *info)
+{
+        data = nvbios_boostSe(bios, idx, data, ver, hdr, cnt, len);
+        memset(info, 0x00, sizeof(*info));
+        if (data) {
+                info->domain  = nv_ro08(bios, data + 0x00);
+                info->percent = nv_ro08(bios, data + 0x01);
+                info->min     = nv_ro16(bios, data + 0x02) * 1000;
+                info->max     = nv_ro16(bios, data + 0x04) * 1000;
+        }
+        return data;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/cstep.c b/drivers/gpu/drm/nouveau/core/subdev/bios/cstep.c
new file mode 100644
index 000000000000..d3b15327fbfd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/cstep.c
@@ -0,0 +1,123 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/cstep.h>
+
+u16
+nvbios_cstepTe(struct nouveau_bios *bios,
+               u8 *ver, u8 *hdr, u8 *cnt, u8 *len, u8 *xnr, u8 *xsz)
+{
+        struct bit_entry bit_P;
+        u16 cstep = 0x0000;
+
+        if (!bit_entry(bios, 'P', &bit_P)) {
+                if (bit_P.version == 2)
+                        cstep = nv_ro16(bios, bit_P.offset + 0x34);
+
+                if (cstep) {
+                        *ver = nv_ro08(bios, cstep + 0);
+                        switch (*ver) {
+                        case 0x10:
+                                *hdr = nv_ro08(bios, cstep + 1);
+                                *cnt = nv_ro08(bios, cstep + 3);
+                                *len = nv_ro08(bios, cstep + 2);
+                                *xnr = nv_ro08(bios, cstep + 5);
+                                *xsz = nv_ro08(bios, cstep + 4);
+                                return cstep;
+                        default:
+                                break;
+                        }
+                }
+        }
+
+        return 0x0000;
+}
+
+u16
+nvbios_cstepEe(struct nouveau_bios *bios, int idx, u8 *ver, u8 *hdr)
+{
+        u8  cnt, len, xnr, xsz;
+        u16 data = nvbios_cstepTe(bios, ver, hdr, &cnt, &len, &xnr, &xsz);
+        if (data && idx < cnt) {
+                data = data + *hdr + (idx * len);
+                *hdr = len;
+                return data;
+        }
+        return 0x0000;
+}
+
+u16
+nvbios_cstepEp(struct nouveau_bios *bios, int idx, u8 *ver, u8 *hdr,
+               struct nvbios_cstepE *info)
+{
+        u16 data = nvbios_cstepEe(bios, idx, ver, hdr);
+        memset(info, 0x00, sizeof(*info));
+        if (data) {
+                info->pstate = (nv_ro16(bios, data + 0x00) & 0x01e0) >> 5;
+                info->index   = nv_ro08(bios, data + 0x03);
+        }
+        return data;
+}
+
+u16
+nvbios_cstepEm(struct nouveau_bios *bios, u8 pstate, u8 *ver, u8 *hdr,
+               struct nvbios_cstepE *info)
+{
+        u32 data, idx = 0;
+        while ((data = nvbios_cstepEp(bios, idx++, ver, hdr, info))) {
+                if (info->pstate == pstate)
+                        break;
+        }
+        return data;
+}
+
+u16
+nvbios_cstepXe(struct nouveau_bios *bios, int idx, u8 *ver, u8 *hdr)
+{
+        u8  cnt, len, xnr, xsz;
+        u16 data = nvbios_cstepTe(bios, ver, hdr, &cnt, &len, &xnr, &xsz);
+        if (data && idx < xnr) {
+                data = data + *hdr + (cnt * len) + (idx * xsz);
+                *hdr = xsz;
+                return data;
+        }
+        return 0x0000;
+}
+
+u16
+nvbios_cstepXp(struct nouveau_bios *bios, int idx, u8 *ver, u8 *hdr,
+               struct nvbios_cstepX *info)
+{
+        u16 data = nvbios_cstepXe(bios, idx, ver, hdr);
+        memset(info, 0x00, sizeof(*info));
+        if (data) {
+                info->freq    = nv_ro16(bios, data + 0x00) * 1000;
+                info->unkn[0] = nv_ro08(bios, data + 0x02);
+                info->unkn[1] = nv_ro08(bios, data + 0x03);
+                info->voltage = nv_ro08(bios, data + 0x04);
+        }
+        return data;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c b/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c
index 663853bcca82..7628fe759220 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/dp.c
@@ -89,6 +89,7 @@ nvbios_dpout_parse(struct nouveau_bios *bios, u8 idx,
                    struct nvbios_dpout *info)
 {
         u16 data = nvbios_dpout_entry(bios, idx, ver, hdr, cnt, len);
+        memset(info, 0x00, sizeof(*info));
         if (data && *ver) {
                 info->type = nv_ro16(bios, data + 0x00);
                 info->mask = nv_ro16(bios, data + 0x02);
@@ -99,9 +100,12 @@ nvbios_dpout_parse(struct nouveau_bios *bios, u8 idx,
                         info->script[0] = nv_ro16(bios, data + 0x06);
                         info->script[1] = nv_ro16(bios, data + 0x08);
                         info->lnkcmp    = nv_ro16(bios, data + 0x0a);
-                        info->script[2] = nv_ro16(bios, data + 0x0c);
-                        info->script[3] = nv_ro16(bios, data + 0x0e);
-                        info->script[4] = nv_ro16(bios, data + 0x10);
+                        if (*len >= 0x0f) {
+                                info->script[2] = nv_ro16(bios, data + 0x0c);
+                                info->script[3] = nv_ro16(bios, data + 0x0e);
+                        }
+                        if (*len >= 0x11)
+                                info->script[4] = nv_ro16(bios, data + 0x10);
                         break;
                 case 0x40:
                         info->flags     = nv_ro08(bios, data + 0x04);
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/init.c b/drivers/gpu/drm/nouveau/core/subdev/bios/init.c
index 57cda2a1437b..420908cb82b6 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/init.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/init.c
@@ -2180,7 +2180,7 @@ nvbios_init(struct nouveau_subdev *subdev, bool execute)
         u16 data;
 
         if (execute)
-                nv_suspend(bios, "running init tables\n");
+                nv_info(bios, "running init tables\n");
         while (!ret && (data = (init_script(bios, ++i)))) {
                 struct nvbios_init init = {
                         .subdev = subdev,
@@ -2210,5 +2210,5 @@ nvbios_init(struct nouveau_subdev *subdev, bool execute)
                 ret = nvbios_exec(&init);
         }
 
-        return 0;
+        return ret;
 }
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/perf.c b/drivers/gpu/drm/nouveau/core/subdev/bios/perf.c
index bcbb056c2887..675e221680aa 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/perf.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/perf.c
@@ -26,8 +26,9 @@
 #include <subdev/bios/bit.h>
 #include <subdev/bios/perf.h>
 
-static u16
-perf_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+u16
+nvbios_perf_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr,
+                  u8 *cnt, u8 *len, u8 *snr, u8 *ssz)
 {
         struct bit_entry bit_P;
         u16 perf = 0x0000;
@@ -38,10 +39,22 @@ perf_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
                         if (perf) {
                                 *ver = nv_ro08(bios, perf + 0);
                                 *hdr = nv_ro08(bios, perf + 1);
+                                if (*ver >= 0x40 && *ver < 0x41) {
+                                        *cnt = nv_ro08(bios, perf + 5);
+                                        *len = nv_ro08(bios, perf + 2);
+                                        *snr = nv_ro08(bios, perf + 4);
+                                        *ssz = nv_ro08(bios, perf + 3);
+                                        return perf;
+                                } else
+                                if (*ver >= 0x20 && *ver < 0x40) {
+                                        *cnt = nv_ro08(bios, perf + 2);
+                                        *len = nv_ro08(bios, perf + 3);
+                                        *snr = nv_ro08(bios, perf + 4);
+                                        *ssz = nv_ro08(bios, perf + 5);
+                                        return perf;
+                                }
                         }
-                } else
-                        nv_error(bios, "unknown offset for perf in BIT P %d\n",
-                                bit_P.version);
+                }
         }
 
         if (bios->bmp_offset) {
@@ -50,19 +63,132 @@ perf_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
                         if (perf) {
                                 *hdr = nv_ro08(bios, perf + 0);
                                 *ver = nv_ro08(bios, perf + 1);
+                                *cnt = nv_ro08(bios, perf + 2);
+                                *len = nv_ro08(bios, perf + 3);
+                                *snr = 0;
+                                *ssz = 0;
+                                return perf;
                         }
                 }
         }
 
+        return 0x0000;
+}
+
+u16
+nvbios_perf_entry(struct nouveau_bios *bios, int idx,
+                  u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+        u8  snr, ssz;
+        u16 perf = nvbios_perf_table(bios, ver, hdr, cnt, len, &snr, &ssz);
+        if (perf && idx < *cnt) {
+                perf = perf + *hdr + (idx * (*len + (snr * ssz)));
+                *hdr = *len;
+                *cnt = snr;
+                *len = ssz;
+                return perf;
+        }
+        return 0x0000;
+}
+
+u16
+nvbios_perfEp(struct nouveau_bios *bios, int idx,
+              u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+              struct nvbios_perfE *info)
+{
+        u16 perf = nvbios_perf_entry(bios, idx, ver, hdr, cnt, len);
+        memset(info, 0x00, sizeof(*info));
+        info->pstate = nv_ro08(bios, perf + 0x00);
+        switch (!!perf * *ver) {
+        case 0x12:
+        case 0x13:
+        case 0x14:
+                info->core     = nv_ro32(bios, perf + 0x01) * 10;
+                info->memory   = nv_ro32(bios, perf + 0x05) * 20;
+                info->fanspeed = nv_ro08(bios, perf + 0x37);
+                if (*hdr > 0x38)
+                        info->voltage = nv_ro08(bios, perf + 0x38);
+                break;
+        case 0x21:
+        case 0x23:
+        case 0x24:
+                info->fanspeed = nv_ro08(bios, perf + 0x04);
+                info->voltage  = nv_ro08(bios, perf + 0x05);
+                info->shader   = nv_ro16(bios, perf + 0x06) * 1000;
+                info->core     = info->shader + (signed char)
+                                 nv_ro08(bios, perf + 0x08) * 1000;
+                switch (nv_device(bios)->chipset) {
+                case 0x49:
+                case 0x4b:
+                        info->memory = nv_ro16(bios, perf + 0x0b) * 1000;
+                        break;
+                default:
+                        info->memory = nv_ro16(bios, perf + 0x0b) * 2000;
+                        break;
+                }
+                break;
+        case 0x25:
+                info->fanspeed = nv_ro08(bios, perf + 0x04);
+                info->voltage  = nv_ro08(bios, perf + 0x05);
+                info->core     = nv_ro16(bios, perf + 0x06) * 1000;
+                info->shader   = nv_ro16(bios, perf + 0x0a) * 1000;
+                info->memory   = nv_ro16(bios, perf + 0x0c) * 1000;
+                break;
+        case 0x30:
+                info->script   = nv_ro16(bios, perf + 0x02);
+        case 0x35:
+                info->fanspeed = nv_ro08(bios, perf + 0x06);
+                info->voltage  = nv_ro08(bios, perf + 0x07);
+                info->core     = nv_ro16(bios, perf + 0x08) * 1000;
+                info->shader   = nv_ro16(bios, perf + 0x0a) * 1000;
+                info->memory   = nv_ro16(bios, perf + 0x0c) * 1000;
+                info->vdec     = nv_ro16(bios, perf + 0x10) * 1000;
+                info->disp     = nv_ro16(bios, perf + 0x14) * 1000;
+                break;
+        case 0x40:
+                info->voltage  = nv_ro08(bios, perf + 0x02);
+                break;
+        default:
+                return 0x0000;
+        }
         return perf;
 }
 
+u32
+nvbios_perfSe(struct nouveau_bios *bios, u32 perfE, int idx,
+              u8 *ver, u8 *hdr, u8 cnt, u8 len)
+{
+        u32 data = 0x00000000;
+        if (idx < cnt) {
+                data = perfE + *hdr + (idx * len);
+                *hdr = len;
+        }
+        return data;
+}
+
+u32
+nvbios_perfSp(struct nouveau_bios *bios, u32 perfE, int idx,
+              u8 *ver, u8 *hdr, u8 cnt, u8 len,
+              struct nvbios_perfS *info)
+{
+        u32 data = nvbios_perfSe(bios, perfE, idx, ver, hdr, cnt, len);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!data * *ver) {
+        case 0x40:
+                info->v40.freq = (nv_ro16(bios, data + 0x00) & 0x3fff) * 1000;
+                break;
+        default:
+                break;
+        }
+        return data;
+}
+
 int
 nvbios_perf_fan_parse(struct nouveau_bios *bios,
                       struct nvbios_perf_fan *fan)
 {
-        u8 ver = 0, hdr = 0, cnt = 0, len = 0;
-        u16 perf = perf_table(bios, &ver, &hdr, &cnt, &len);
+        u8  ver, hdr, cnt, len, snr, ssz;
+        u16 perf = nvbios_perf_table(bios, &ver, &hdr, &cnt, &len, &snr, &ssz);
         if (!perf)
                 return -ENODEV;
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/pll.c b/drivers/gpu/drm/nouveau/core/subdev/bios/pll.c
index f835501203e5..1f76de597d4b 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/pll.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/pll.c
@@ -114,6 +114,7 @@ pll_map(struct nouveau_bios *bios)
         switch (nv_device(bios)->card_type) {
         case NV_04:
         case NV_10:
+        case NV_11:
         case NV_20:
         case NV_30:
                 return nv04_pll_mapping;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/rammap.c b/drivers/gpu/drm/nouveau/core/subdev/bios/rammap.c
new file mode 100644
index 000000000000..916fa9d302b7
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/rammap.c
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/rammap.h>
+
+u16
+nvbios_rammap_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr,
+                    u8 *cnt, u8 *len, u8 *snr, u8 *ssz)
+{
+        struct bit_entry bit_P;
+        u16 rammap = 0x0000;
+
+        if (!bit_entry(bios, 'P', &bit_P)) {
+                if (bit_P.version == 2)
+                        rammap = nv_ro16(bios, bit_P.offset + 4);
+
+                if (rammap) {
+                        *ver = nv_ro08(bios, rammap + 0);
+                        switch (*ver) {
+                        case 0x10:
+                        case 0x11:
+                                *hdr = nv_ro08(bios, rammap + 1);
+                                *cnt = nv_ro08(bios, rammap + 5);
+                                *len = nv_ro08(bios, rammap + 2);
+                                *snr = nv_ro08(bios, rammap + 4);
+                                *ssz = nv_ro08(bios, rammap + 3);
+                                return rammap;
+                        default:
+                                break;
+                        }
+                }
+        }
+
+        return 0x0000;
+}
+
+u16
+nvbios_rammap_entry(struct nouveau_bios *bios, int idx,
+                    u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+        u8  snr, ssz;
+        u16 rammap = nvbios_rammap_table(bios, ver, hdr, cnt, len, &snr, &ssz);
+        if (rammap && idx < *cnt) {
+                rammap = rammap + *hdr + (idx * (*len + (snr * ssz)));
+                *hdr = *len;
+                *cnt = snr;
+                *len = ssz;
+                return rammap;
+        }
+        return 0x0000;
+}
+
+u16
+nvbios_rammap_match(struct nouveau_bios *bios, u16 khz,
+                    u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+        int idx = 0;
+        u32 data;
+        while ((data = nvbios_rammap_entry(bios, idx++, ver, hdr, cnt, len))) {
+                if (khz >= nv_ro16(bios, data + 0x00) &&
+                    khz <= nv_ro16(bios, data + 0x02))
+                        break;
+        }
+        return data;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/timing.c b/drivers/gpu/drm/nouveau/core/subdev/bios/timing.c
new file mode 100644
index 000000000000..151c2d6aaee8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/timing.c
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/timing.h>
+
+u16
+nvbios_timing_table(struct nouveau_bios *bios,
+                    u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+        struct bit_entry bit_P;
+        u16 timing = 0x0000;
+
+        if (!bit_entry(bios, 'P', &bit_P)) {
+                if (bit_P.version == 1)
+                        timing = nv_ro16(bios, bit_P.offset + 4);
+                else
+                if (bit_P.version == 2)
+                        timing = nv_ro16(bios, bit_P.offset + 8);
+
+                if (timing) {
+                        *ver = nv_ro08(bios, timing + 0);
+                        switch (*ver) {
+                        case 0x10:
+                                *hdr = nv_ro08(bios, timing + 1);
+                                *cnt = nv_ro08(bios, timing + 2);
+                                *len = nv_ro08(bios, timing + 3);
+                                return timing;
+                        case 0x20:
+                                *hdr = nv_ro08(bios, timing + 1);
+                                *cnt = nv_ro08(bios, timing + 3);
+                                *len = nv_ro08(bios, timing + 2);
+                                return timing;
+                        default:
+                                break;
+                        }
+                }
+        }
+
+        return 0x0000;
+}
+
+u16
+nvbios_timing_entry(struct nouveau_bios *bios, int idx, u8 *ver, u8 *len)
+{
+        u8  hdr, cnt;
+        u16 timing = nvbios_timing_table(bios, ver, &hdr, &cnt, len);
+        if (timing && idx < cnt)
+                return timing + hdr + (idx * *len);
+        return 0x0000;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/vmap.c b/drivers/gpu/drm/nouveau/core/subdev/bios/vmap.c
new file mode 100644
index 000000000000..f343a1b060e8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/vmap.c
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2012 Nouveau Community
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Martin Peres
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/vmap.h>
+
+u16
+nvbios_vmap_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+        struct bit_entry bit_P;
+        u16 vmap = 0x0000;
+
+        if (!bit_entry(bios, 'P', &bit_P)) {
+                if (bit_P.version == 2) {
+                        vmap = nv_ro16(bios, bit_P.offset + 0x20);
+                        if (vmap) {
+                                *ver = nv_ro08(bios, vmap + 0);
+                                switch (*ver) {
+                                case 0x10:
+                                case 0x20:
+                                        *hdr = nv_ro08(bios, vmap + 1);
+                                        *cnt = nv_ro08(bios, vmap + 3);
+                                        *len = nv_ro08(bios, vmap + 2);
+                                        return vmap;
+                                default:
+                                        break;
+                                }
+                        }
+                }
+        }
+
+        return 0x0000;
+}
+
+u16
+nvbios_vmap_parse(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+                  struct nvbios_vmap *info)
+{
+        u16 vmap = nvbios_vmap_table(bios, ver, hdr, cnt, len);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!vmap * *ver) {
+        case 0x10:
+        case 0x20:
+                break;
+        }
+        return vmap;
+}
+
+u16
+nvbios_vmap_entry(struct nouveau_bios *bios, int idx, u8 *ver, u8 *len)
+{
+        u8  hdr, cnt;
+        u16 vmap = nvbios_vmap_table(bios, ver, &hdr, &cnt, len);
+        if (vmap && idx < cnt) {
+                vmap = vmap + hdr + (idx * *len);
+                return vmap;
+        }
+        return 0x0000;
+}
+
+u16
+nvbios_vmap_entry_parse(struct nouveau_bios *bios, int idx, u8 *ver, u8 *len,
+                        struct nvbios_vmap_entry *info)
+{
+        u16 vmap = nvbios_vmap_entry(bios, idx, ver, len);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!vmap * *ver) {
+        case 0x10:
+                info->link   = 0xff;
+                info->min    = nv_ro32(bios, vmap + 0x00);
+                info->max    = nv_ro32(bios, vmap + 0x04);
+                info->arg[0] = nv_ro32(bios, vmap + 0x08);
+                info->arg[1] = nv_ro32(bios, vmap + 0x0c);
+                info->arg[2] = nv_ro32(bios, vmap + 0x10);
+                break;
+        case 0x20:
+                info->unk0   = nv_ro08(bios, vmap + 0x00);
+                info->link   = nv_ro08(bios, vmap + 0x01);
+                info->min    = nv_ro32(bios, vmap + 0x02);
+                info->max    = nv_ro32(bios, vmap + 0x06);
+                info->arg[0] = nv_ro32(bios, vmap + 0x0a);
+                info->arg[1] = nv_ro32(bios, vmap + 0x0e);
+                info->arg[2] = nv_ro32(bios, vmap + 0x12);
+                info->arg[3] = nv_ro32(bios, vmap + 0x16);
+                info->arg[4] = nv_ro32(bios, vmap + 0x1a);
+                info->arg[5] = nv_ro32(bios, vmap + 0x1e);
+                break;
+        }
+        return vmap;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/volt.c b/drivers/gpu/drm/nouveau/core/subdev/bios/volt.c
new file mode 100644
index 000000000000..bb590de4ecb2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/volt.c
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2012 Nouveau Community
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Martin Peres
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/volt.h>
+
+u16
+nvbios_volt_table(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
+{
+        struct bit_entry bit_P;
+        u16 volt = 0x0000;
+
+        if (!bit_entry(bios, 'P', &bit_P)) {
+                if (bit_P.version == 2)
+                        volt = nv_ro16(bios, bit_P.offset + 0x0c);
+                else
+                if (bit_P.version == 1)
+                        volt = nv_ro16(bios, bit_P.offset + 0x10);
+
+                if (volt) {
+                        *ver = nv_ro08(bios, volt + 0);
+                        switch (*ver) {
+                        case 0x12:
+                                *hdr = 5;
+                                *cnt = nv_ro08(bios, volt + 2);
+                                *len = nv_ro08(bios, volt + 1);
+                                return volt;
+                        case 0x20:
+                                *hdr = nv_ro08(bios, volt + 1);
+                                *cnt = nv_ro08(bios, volt + 2);
+                                *len = nv_ro08(bios, volt + 3);
+                                return volt;
+                        case 0x30:
+                        case 0x40:
+                        case 0x50:
+                                *hdr = nv_ro08(bios, volt + 1);
+                                *cnt = nv_ro08(bios, volt + 3);
+                                *len = nv_ro08(bios, volt + 2);
+                                return volt;
+                        }
+                }
+        }
+
+        return 0x0000;
+}
+
+u16
+nvbios_volt_parse(struct nouveau_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len,
+                  struct nvbios_volt *info)
+{
+        u16 volt = nvbios_volt_table(bios, ver, hdr, cnt, len);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!volt * *ver) {
+        case 0x12:
+                info->vidmask = nv_ro08(bios, volt + 0x04);
+                break;
+        case 0x20:
+                info->vidmask = nv_ro08(bios, volt + 0x05);
+                break;
+        case 0x30:
+                info->vidmask = nv_ro08(bios, volt + 0x04);
+                break;
+        case 0x40:
+                info->base    = nv_ro32(bios, volt + 0x04);
+                info->step    = nv_ro16(bios, volt + 0x08);
+                info->vidmask = nv_ro08(bios, volt + 0x0b);
+                /*XXX*/
+                info->min     = 0;
+                info->max     = info->base;
+                break;
+        case 0x50:
+                info->vidmask = nv_ro08(bios, volt + 0x06);
+                info->min     = nv_ro32(bios, volt + 0x0a);
+                info->max     = nv_ro32(bios, volt + 0x0e);
+                info->base    = nv_ro32(bios, volt + 0x12) & 0x00ffffff;
+                info->step    = nv_ro16(bios, volt + 0x16);
+                break;
+        }
+        return volt;
+}
+
+u16
+nvbios_volt_entry(struct nouveau_bios *bios, int idx, u8 *ver, u8 *len)
+{
+        u8  hdr, cnt;
+        u16 volt = nvbios_volt_table(bios, ver, &hdr, &cnt, len);
+        if (volt && idx < cnt) {
+                volt = volt + hdr + (idx * *len);
+                return volt;
+        }
+        return 0x0000;
+}
+
+u16
+nvbios_volt_entry_parse(struct nouveau_bios *bios, int idx, u8 *ver, u8 *len,
+                        struct nvbios_volt_entry *info)
+{
+        u16 volt = nvbios_volt_entry(bios, idx, ver, len);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!volt * *ver) {
+        case 0x12:
+        case 0x20:
+                info->voltage = nv_ro08(bios, volt + 0x00) * 10000;
+                info->vid     = nv_ro08(bios, volt + 0x01);
+                break;
+        case 0x30:
+                info->voltage = nv_ro08(bios, volt + 0x00) * 10000;
+                info->vid     = nv_ro08(bios, volt + 0x01) >> 2;
+                break;
+        case 0x40:
+        case 0x50:
+                break;
+        }
+        return volt;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bus/hwsq.c b/drivers/gpu/drm/nouveau/core/subdev/bus/hwsq.c
new file mode 100644
index 000000000000..f757470e2284
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bus/hwsq.c
@@ -0,0 +1,145 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include <subdev/timer.h>
+#include <subdev/bus.h>
+
+struct nouveau_hwsq {
+        struct nouveau_bus *pbus;
+        u32 addr;
+        u32 data;
+        struct {
+                u8 data[512];
+                u8 size;
+        } c;
+};
+
+static void
+hwsq_cmd(struct nouveau_hwsq *hwsq, int size, u8 data[])
+{
+        memcpy(&hwsq->c.data[hwsq->c.size], data, size * sizeof(data[0]));
+        hwsq->c.size += size;
+}
+
+int
+nouveau_hwsq_init(struct nouveau_bus *pbus, struct nouveau_hwsq **phwsq)
+{
+        struct nouveau_hwsq *hwsq;
+
+        hwsq = *phwsq = kmalloc(sizeof(*hwsq), GFP_KERNEL);
+        if (hwsq) {
+                hwsq->pbus = pbus;
+                hwsq->addr = ~0;
+                hwsq->data = ~0;
+                memset(hwsq->c.data, 0x7f, sizeof(hwsq->c.data));
+                hwsq->c.size = 0;
+        }
+
+        return hwsq ? 0 : -ENOMEM;
+}
+
+int
+nouveau_hwsq_fini(struct nouveau_hwsq **phwsq, bool exec)
+{
+        struct nouveau_hwsq *hwsq = *phwsq;
+        int ret = 0, i;
+        if (hwsq) {
+                struct nouveau_bus *pbus = hwsq->pbus;
+                hwsq->c.size = (hwsq->c.size + 4) / 4;
+                if (hwsq->c.size <= pbus->hwsq_size) {
+                        if (exec)
+                                ret = pbus->hwsq_exec(pbus, (u32 *)hwsq->c.data,
+                                                      hwsq->c.size);
+                        if (ret)
+                                nv_error(pbus, "hwsq exec failed: %d\n", ret);
+                } else {
+                        nv_error(pbus, "hwsq ucode too large\n");
+                        ret = -ENOSPC;
+                }
+
+                for (i = 0; ret && i < hwsq->c.size; i++)
+                        nv_error(pbus, "\t0x%08x\n", ((u32 *)hwsq->c.data)[i]);
+
+                *phwsq = NULL;
+                kfree(hwsq);
+        }
+        return ret;
+}
+
+void
+nouveau_hwsq_wr32(struct nouveau_hwsq *hwsq, u32 addr, u32 data)
+{
+        nv_debug(hwsq->pbus, "R[%06x] = 0x%08x\n", addr, data);
+
+        if (hwsq->data != data) {
+                if ((data & 0xffff0000) != (hwsq->data & 0xffff0000)) {
+                        hwsq_cmd(hwsq, 5, (u8[]){ 0xe2, data, data >> 8,
+                                                  data >> 16, data >> 24 });
+                } else {
+                        hwsq_cmd(hwsq, 3, (u8[]){ 0x42, data, data >> 8 });
+                }
+        }
+
+        if ((addr & 0xffff0000) != (hwsq->addr & 0xffff0000)) {
+                hwsq_cmd(hwsq, 5, (u8[]){ 0xe0, addr, addr >> 8,
+                                          addr >> 16, addr >> 24 });
+        } else {
+                hwsq_cmd(hwsq, 3, (u8[]){ 0x40, addr, addr >> 8 });
+        }
+
+        hwsq->addr = addr;
+        hwsq->data = data;
+}
+
+void
+nouveau_hwsq_setf(struct nouveau_hwsq *hwsq, u8 flag, int data)
+{
+        nv_debug(hwsq->pbus, " FLAG[%02x] = %d\n", flag, data);
+        flag += 0x80;
+        if (data >= 0)
+                flag += 0x20;
+        if (data >= 1)
+                flag += 0x20;
+        hwsq_cmd(hwsq, 1, (u8[]){ flag });
+}
+
+void
+nouveau_hwsq_wait(struct nouveau_hwsq *hwsq, u8 flag, u8 data)
+{
+        nv_debug(hwsq->pbus, " WAIT[%02x] = %d\n", flag, data);
+        hwsq_cmd(hwsq, 3, (u8[]){ 0x5f, flag, data });
+}
+
+void
+nouveau_hwsq_nsec(struct nouveau_hwsq *hwsq, u32 nsec)
+{
+        u8 shift = 0, usec = nsec / 1000;
+        while (usec & ~3) {
+                usec >>= 2;
+                shift++;
+        }
+
+        nv_debug(hwsq->pbus, "    DELAY = %d ns\n", nsec);
+        hwsq_cmd(hwsq, 1, (u8[]){ 0x00 | (shift << 2) | usec });
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bus/hwsq.h b/drivers/gpu/drm/nouveau/core/subdev/bus/hwsq.h
new file mode 100644
index 000000000000..12176f9c1bc6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bus/hwsq.h
@@ -0,0 +1,113 @@
+#ifndef __NVKM_BUS_HWSQ_H__
+#define __NVKM_BUS_HWSQ_H__
+
+#include <subdev/bus.h>
+
+struct hwsq {
+        struct nouveau_subdev *subdev;
+        struct nouveau_hwsq *hwsq;
+        int sequence;
+};
+
+struct hwsq_reg {
+        int sequence;
+        bool force;
+        u32 addr[2];
+        u32 data;
+};
+
+static inline struct hwsq_reg
+hwsq_reg2(u32 addr1, u32 addr2)
+{
+        return (struct hwsq_reg) {
+                .sequence = 0,
+                .force = 0,
+                .addr = { addr1, addr2 },
+                .data = 0xdeadbeef,
+        };
+}
+
+static inline struct hwsq_reg
+hwsq_reg(u32 addr)
+{
+        return hwsq_reg2(addr, addr);
+}
+
+static inline int
+hwsq_init(struct hwsq *ram, struct nouveau_subdev *subdev)
+{
+        struct nouveau_bus *pbus = nouveau_bus(subdev);
+        int ret;
+
+        ret = nouveau_hwsq_init(pbus, &ram->hwsq);
+        if (ret)
+                return ret;
+
+        ram->sequence++;
+        ram->subdev = subdev;
+        return 0;
+}
+
+static inline int
+hwsq_exec(struct hwsq *ram, bool exec)
+{
+        int ret = 0;
+        if (ram->subdev) {
+                ret = nouveau_hwsq_fini(&ram->hwsq, exec);
+                ram->subdev = NULL;
+        }
+        return ret;
+}
+
+static inline u32
+hwsq_rd32(struct hwsq *ram, struct hwsq_reg *reg)
+{
+        if (reg->sequence != ram->sequence)
+                reg->data = nv_rd32(ram->subdev, reg->addr[0]);
+        return reg->data;
+}
+
+static inline void
+hwsq_wr32(struct hwsq *ram, struct hwsq_reg *reg, u32 data)
+{
+        reg->sequence = ram->sequence;
+        reg->data = data;
+        if (reg->addr[0] != reg->addr[1])
+                nouveau_hwsq_wr32(ram->hwsq, reg->addr[1], reg->data);
+        nouveau_hwsq_wr32(ram->hwsq, reg->addr[0], reg->data);
+}
+
+static inline void
+hwsq_nuke(struct hwsq *ram, struct hwsq_reg *reg)
+{
+        reg->force = true;
+}
+
+static inline u32
+hwsq_mask(struct hwsq *ram, struct hwsq_reg *reg, u32 mask, u32 data)
+{
+        u32 temp = hwsq_rd32(ram, reg);
+        if (temp != ((temp & ~mask) | data) || reg->force)
+                hwsq_wr32(ram, reg, (temp & ~mask) | data);
+        return temp;
+}
+
+static inline void
+hwsq_setf(struct hwsq *ram, u8 flag, int data)
+{
+        nouveau_hwsq_setf(ram->hwsq, flag, data);
+}
+
+static inline void
+hwsq_wait(struct hwsq *ram, u8 flag, u8 data)
+{
+        nouveau_hwsq_wait(ram->hwsq, flag, data);
+}
+
+static inline void
+hwsq_nsec(struct hwsq *ram, u32 nsec)
+{
+        nouveau_hwsq_nsec(ram->hwsq, nsec);
+}
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bus/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/bus/nv04.c
index 8c7f8057a185..23921b5351db 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bus/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bus/nv04.c
@@ -23,11 +23,7 @@
  *          Ben Skeggs
  */
 
-#include <subdev/bus.h>
-
-struct nv04_bus_priv {
-        struct nouveau_bus base;
-};
+#include "nv04.h"
 
 static void
 nv04_bus_intr(struct nouveau_subdev *subdev)
@@ -56,10 +52,22 @@ nv04_bus_intr(struct nouveau_subdev *subdev)
 }
 
 static int
+nv04_bus_init(struct nouveau_object *object)
+{
+        struct nv04_bus_priv *priv = (void *)object;
+
+        nv_wr32(priv, 0x001100, 0xffffffff);
+        nv_wr32(priv, 0x001140, 0x00000111);
+
+        return nouveau_bus_init(&priv->base);
+}
+
+int
 nv04_bus_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
               struct nouveau_oclass *oclass, void *data, u32 size,
               struct nouveau_object **pobject)
 {
+        struct nv04_bus_impl *impl = (void *)oclass;
         struct nv04_bus_priv *priv;
         int ret;
 
@@ -68,28 +76,20 @@ nv04_bus_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        nv_subdev(priv)->intr = nv04_bus_intr;
+        nv_subdev(priv)->intr = impl->intr;
+        priv->base.hwsq_exec = impl->hwsq_exec;
+        priv->base.hwsq_size = impl->hwsq_size;
         return 0;
 }
 
-static int
-nv04_bus_init(struct nouveau_object *object)
-{
-        struct nv04_bus_priv *priv = (void *)object;
-
-        nv_wr32(priv, 0x001100, 0xffffffff);
-        nv_wr32(priv, 0x001140, 0x00000111);
-
-        return nouveau_bus_init(&priv->base);
-}
-
-struct nouveau_oclass
-nv04_bus_oclass = {
-        .handle = NV_SUBDEV(BUS, 0x04),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nv04_bus_oclass = &(struct nv04_bus_impl) {
+        .base.handle = NV_SUBDEV(BUS, 0x04),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv04_bus_ctor,
                 .dtor = _nouveau_bus_dtor,
                 .init = nv04_bus_init,
                 .fini = _nouveau_bus_fini,
         },
-};
+        .intr = nv04_bus_intr,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bus/nv04.h b/drivers/gpu/drm/nouveau/core/subdev/bus/nv04.h
new file mode 100644
index 000000000000..4d7602450a20
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bus/nv04.h
@@ -0,0 +1,23 @@
+#ifndef __NVKM_BUS_NV04_H__
+#define __NVKM_BUS_NV04_H__
+
+#include <subdev/bus.h>
+
+struct nv04_bus_priv {
+        struct nouveau_bus base;
+};
+
+int  nv04_bus_ctor(struct nouveau_object *, struct nouveau_object *,
+                   struct nouveau_oclass *, void *, u32,
+                   struct nouveau_object **);
+int  nv50_bus_init(struct nouveau_object *);
+void nv50_bus_intr(struct nouveau_subdev *);
+
+struct nv04_bus_impl {
+        struct nouveau_oclass base;
+        void (*intr)(struct nouveau_subdev *);
+        int  (*hwsq_exec)(struct nouveau_bus *, u32 *, u32);
+        u32  hwsq_size;
+};
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bus/nv31.c b/drivers/gpu/drm/nouveau/core/subdev/bus/nv31.c
index 34132aef34e1..94da46f61627 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bus/nv31.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bus/nv31.c
@@ -23,11 +23,7 @@
  *          Ben Skeggs
  */
 
-#include <subdev/bus.h>
-
-struct nv31_bus_priv {
-        struct nouveau_bus base;
-};
+#include "nv04.h"
 
 static void
 nv31_bus_intr(struct nouveau_subdev *subdev)
@@ -71,7 +67,7 @@ nv31_bus_intr(struct nouveau_subdev *subdev)
 static int
 nv31_bus_init(struct nouveau_object *object)
 {
-        struct nv31_bus_priv *priv = (void *)object;
+        struct nv04_bus_priv *priv = (void *)object;
         int ret;
 
         ret = nouveau_bus_init(&priv->base);
@@ -83,30 +79,14 @@ nv31_bus_init(struct nouveau_object *object)
         return 0;
 }
 
-static int
-nv31_bus_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-              struct nouveau_oclass *oclass, void *data, u32 size,
-              struct nouveau_object **pobject)
-{
-        struct nv31_bus_priv *priv;
-        int ret;
-
-        ret = nouveau_bus_create(parent, engine, oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        nv_subdev(priv)->intr = nv31_bus_intr;
-        return 0;
-}
-
-struct nouveau_oclass
-nv31_bus_oclass = {
-        .handle = NV_SUBDEV(BUS, 0x31),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv31_bus_ctor,
+struct nouveau_oclass *
+nv31_bus_oclass = &(struct nv04_bus_impl) {
+        .base.handle = NV_SUBDEV(BUS, 0x31),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_bus_ctor,
                 .dtor = _nouveau_bus_dtor,
                 .init = nv31_bus_init,
                 .fini = _nouveau_bus_fini,
         },
-};
+        .intr = nv31_bus_intr,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bus/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/bus/nv50.c
index f5b2117fa8c6..11918f7e2aca 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bus/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bus/nv50.c
@@ -23,13 +23,27 @@
  *          Ben Skeggs
  */
 
-#include <subdev/bus.h>
+#include <subdev/timer.h>
 
-struct nv50_bus_priv {
-        struct nouveau_bus base;
-};
+#include "nv04.h"
 
-static void
+static int
+nv50_bus_hwsq_exec(struct nouveau_bus *pbus, u32 *data, u32 size)
+{
+        struct nv50_bus_priv *priv = (void *)pbus;
+        int i;
+
+        nv_mask(pbus, 0x001098, 0x00000008, 0x00000000);
+        nv_wr32(pbus, 0x001304, 0x00000000);
+        for (i = 0; i < size; i++)
+                nv_wr32(priv, 0x001400 + (i * 4), data[i]);
+        nv_mask(pbus, 0x001098, 0x00000018, 0x00000018);
+        nv_wr32(pbus, 0x00130c, 0x00000003);
+
+        return nv_wait(pbus, 0x001308, 0x00000100, 0x00000000) ? 0 : -ETIMEDOUT;
+}
+
+void
 nv50_bus_intr(struct nouveau_subdev *subdev)
 {
         struct nouveau_bus *pbus = nouveau_bus(subdev);
@@ -61,10 +75,10 @@ nv50_bus_intr(struct nouveau_subdev *subdev)
         }
 }
 
-static int
+int
 nv50_bus_init(struct nouveau_object *object)
 {
-        struct nv50_bus_priv *priv = (void *)object;
+        struct nv04_bus_priv *priv = (void *)object;
         int ret;
 
         ret = nouveau_bus_init(&priv->base);
@@ -76,30 +90,16 @@ nv50_bus_init(struct nouveau_object *object)
         return 0;
 }
 
-static int
-nv50_bus_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-              struct nouveau_oclass *oclass, void *data, u32 size,
-              struct nouveau_object **pobject)
-{
-        struct nv50_bus_priv *priv;
-        int ret;
-
-        ret = nouveau_bus_create(parent, engine, oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        nv_subdev(priv)->intr = nv50_bus_intr;
-        return 0;
-}
-
-struct nouveau_oclass
-nv50_bus_oclass = {
-        .handle = NV_SUBDEV(BUS, 0x50),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv50_bus_ctor,
+struct nouveau_oclass *
+nv50_bus_oclass = &(struct nv04_bus_impl) {
+        .base.handle = NV_SUBDEV(BUS, 0x50),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_bus_ctor,
                 .dtor = _nouveau_bus_dtor,
                 .init = nv50_bus_init,
                 .fini = _nouveau_bus_fini,
         },
-};
+        .intr = nv50_bus_intr,
+        .hwsq_exec = nv50_bus_hwsq_exec,
+        .hwsq_size = 64,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bus/nv94.c b/drivers/gpu/drm/nouveau/core/subdev/bus/nv94.c
new file mode 100644
index 000000000000..d3659055fa4b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bus/nv94.c
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2012 Nouveau Community
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Martin Peres <martin.peres@labri.fr>
+ *          Ben Skeggs
+ */
+
+#include <subdev/timer.h>
+
+#include "nv04.h"
+
+static int
+nv94_bus_hwsq_exec(struct nouveau_bus *pbus, u32 *data, u32 size)
+{
+        struct nv50_bus_priv *priv = (void *)pbus;
+        int i;
+
+        nv_mask(pbus, 0x001098, 0x00000008, 0x00000000);
+        nv_wr32(pbus, 0x001304, 0x00000000);
+        nv_wr32(pbus, 0x001318, 0x00000000);
+        for (i = 0; i < size; i++)
+                nv_wr32(priv, 0x080000 + (i * 4), data[i]);
+        nv_mask(pbus, 0x001098, 0x00000018, 0x00000018);
+        nv_wr32(pbus, 0x00130c, 0x00000001);
+
+        return nv_wait(pbus, 0x001308, 0x00000100, 0x00000000) ? 0 : -ETIMEDOUT;
+}
+
+struct nouveau_oclass *
+nv94_bus_oclass = &(struct nv04_bus_impl) {
+        .base.handle = NV_SUBDEV(BUS, 0x94),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_bus_ctor,
+                .dtor = _nouveau_bus_dtor,
+                .init = nv50_bus_init,
+                .fini = _nouveau_bus_fini,
+        },
+        .intr = nv50_bus_intr,
+        .hwsq_exec = nv94_bus_hwsq_exec,
+        .hwsq_size = 128,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bus/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/bus/nvc0.c
index b192d6246363..73839d7151a7 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bus/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bus/nvc0.c
@@ -23,11 +23,7 @@
  *          Ben Skeggs
  */
 
-#include <subdev/bus.h>
-
-struct nvc0_bus_priv {
-        struct nouveau_bus base;
-};
+#include "nv04.h"
 
 static void
 nvc0_bus_intr(struct nouveau_subdev *subdev)
@@ -60,7 +56,7 @@ nvc0_bus_intr(struct nouveau_subdev *subdev)
 static int
 nvc0_bus_init(struct nouveau_object *object)
 {
-        struct nvc0_bus_priv *priv = (void *)object;
+        struct nv04_bus_priv *priv = (void *)object;
         int ret;
 
         ret = nouveau_bus_init(&priv->base);
@@ -72,30 +68,14 @@ nvc0_bus_init(struct nouveau_object *object)
         return 0;
 }
 
-static int
-nvc0_bus_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-              struct nouveau_oclass *oclass, void *data, u32 size,
-              struct nouveau_object **pobject)
-{
-        struct nvc0_bus_priv *priv;
-        int ret;
-
-        ret = nouveau_bus_create(parent, engine, oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        nv_subdev(priv)->intr = nvc0_bus_intr;
-        return 0;
-}
-
-struct nouveau_oclass
-nvc0_bus_oclass = {
-        .handle = NV_SUBDEV(BUS, 0xc0),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nvc0_bus_ctor,
+struct nouveau_oclass *
+nvc0_bus_oclass = &(struct nv04_bus_impl) {
+        .base.handle = NV_SUBDEV(BUS, 0xc0),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_bus_ctor,
                 .dtor = _nouveau_bus_dtor,
                 .init = nvc0_bus_init,
                 .fini = _nouveau_bus_fini,
         },
-};
+        .intr = nvc0_bus_intr,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/base.c b/drivers/gpu/drm/nouveau/core/subdev/clock/base.c
new file mode 100644
index 000000000000..e2938a21b06f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/base.c
@@ -0,0 +1,494 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <core/option.h>
+
+#include <subdev/clock.h>
+#include <subdev/therm.h>
+#include <subdev/volt.h>
+#include <subdev/fb.h>
+
+#include <subdev/bios.h>
+#include <subdev/bios/boost.h>
+#include <subdev/bios/cstep.h>
+#include <subdev/bios/perf.h>
+
+/******************************************************************************
+ * misc
+ *****************************************************************************/
+static u32
+nouveau_clock_adjust(struct nouveau_clock *clk, bool adjust,
+                     u8 pstate, u8 domain, u32 input)
+{
+        struct nouveau_bios *bios = nouveau_bios(clk);
+        struct nvbios_boostE boostE;
+        u8  ver, hdr, cnt, len;
+        u16 data;
+
+        data = nvbios_boostEm(bios, pstate, &ver, &hdr, &cnt, &len, &boostE);
+        if (data) {
+                struct nvbios_boostS boostS;
+                u8  idx = 0, sver, shdr;
+                u16 subd;
+
+                input = max(boostE.min, input);
+                input = min(boostE.max, input);
+                do {
+                        sver = ver;
+                        shdr = hdr;
+                        subd = nvbios_boostSp(bios, idx++, data, &sver, &shdr,
+                                              cnt, len, &boostS);
+                        if (subd && boostS.domain == domain) {
+                                if (adjust)
+                                        input = input * boostS.percent / 100;
+                                input = max(boostS.min, input);
+                                input = min(boostS.max, input);
+                                break;
+                        }
+                } while (subd);
+        }
+
+        return input;
+}
+
+/******************************************************************************
+ * C-States
+ *****************************************************************************/
+static int
+nouveau_cstate_prog(struct nouveau_clock *clk,
+                    struct nouveau_pstate *pstate, int cstatei)
+{
+        struct nouveau_therm *ptherm = nouveau_therm(clk);
+        struct nouveau_volt *volt = nouveau_volt(clk);
+        struct nouveau_cstate *cstate;
+        int ret;
+
+        if (!list_empty(&pstate->list)) {
+                cstate = list_entry(pstate->list.prev, typeof(*cstate), head);
+        } else {
+                cstate = &pstate->base;
+        }
+
+        ret = nouveau_therm_cstate(ptherm, pstate->fanspeed, +1);
+        if (ret && ret != -ENODEV) {
+                nv_error(clk, "failed to raise fan speed: %d\n", ret);
+                return ret;
+        }
+
+        ret = volt->set_id(volt, cstate->voltage, +1);
+        if (ret && ret != -ENODEV) {
+                nv_error(clk, "failed to raise voltage: %d\n", ret);
+                return ret;
+        }
+
+        ret = clk->calc(clk, cstate);
+        if (ret == 0) {
+                ret = clk->prog(clk);
+                clk->tidy(clk);
+        }
+
+        ret = volt->set_id(volt, cstate->voltage, -1);
+        if (ret && ret != -ENODEV)
+                nv_error(clk, "failed to lower voltage: %d\n", ret);
+
+        ret = nouveau_therm_cstate(ptherm, pstate->fanspeed, -1);
+        if (ret && ret != -ENODEV)
+                nv_error(clk, "failed to lower fan speed: %d\n", ret);
+
+        return 0;
+}
+
+static void
+nouveau_cstate_del(struct nouveau_cstate *cstate)
+{
+        list_del(&cstate->head);
+        kfree(cstate);
+}
+
+static int
+nouveau_cstate_new(struct nouveau_clock *clk, int idx,
+                   struct nouveau_pstate *pstate)
+{
+        struct nouveau_bios *bios = nouveau_bios(clk);
+        struct nouveau_clocks *domain = clk->domains;
+        struct nouveau_cstate *cstate = NULL;
+        struct nvbios_cstepX cstepX;
+        u8  ver, hdr;
+        u16 data;
+
+        data = nvbios_cstepXp(bios, idx, &ver, &hdr, &cstepX);
+        if (!data)
+                return -ENOENT;
+
+        cstate = kzalloc(sizeof(*cstate), GFP_KERNEL);
+        if (!cstate)
+                return -ENOMEM;
+
+        *cstate = pstate->base;
+        cstate->voltage = cstepX.voltage;
+
+        while (domain && domain->name != nv_clk_src_max) {
+                if (domain->flags & NVKM_CLK_DOM_FLAG_CORE) {
+                        u32 freq = nouveau_clock_adjust(clk, true,
+                                                        pstate->pstate,
+                                                        domain->bios,
+                                                        cstepX.freq);
+                        cstate->domain[domain->name] = freq;
+                }
+                domain++;
+        }
+
+        list_add(&cstate->head, &pstate->list);
+        return 0;
+}
+
+/******************************************************************************
+ * P-States
+ *****************************************************************************/
+static int
+nouveau_pstate_prog(struct nouveau_clock *clk, int pstatei)
+{
+        struct nouveau_fb *pfb = nouveau_fb(clk);
+        struct nouveau_pstate *pstate;
+        int ret, idx = 0;
+
+        list_for_each_entry(pstate, &clk->states, head) {
+                if (idx++ == pstatei)
+                        break;
+        }
+
+        nv_debug(clk, "setting performance state %d\n", pstatei);
+        clk->pstate = pstatei;
+
+        if (pfb->ram->calc) {
+                ret = pfb->ram->calc(pfb, pstate->base.domain[nv_clk_src_mem]);
+                if (ret == 0)
+                        ret = pfb->ram->prog(pfb);
+                pfb->ram->tidy(pfb);
+        }
+
+        return nouveau_cstate_prog(clk, pstate, 0);
+}
+
+static int
+nouveau_pstate_calc(struct nouveau_clock *clk)
+{
+        int pstate, ret = 0;
+
+        nv_trace(clk, "P %d U %d A %d T %d D %d\n", clk->pstate,
+                 clk->ustate, clk->astate, clk->tstate, clk->dstate);
+
+        if (clk->state_nr && clk->ustate != -1) {
+                pstate = (clk->ustate < 0) ? clk->astate : clk->ustate;
+                pstate = min(pstate, clk->state_nr - 1 - clk->tstate);
+                pstate = max(pstate, clk->dstate);
+        } else {
+                pstate = clk->pstate = -1;
+        }
+
+        nv_trace(clk, "-> %d\n", pstate);
+        if (pstate != clk->pstate)
+                ret = nouveau_pstate_prog(clk, pstate);
+        return ret;
+}
+
+static void
+nouveau_pstate_info(struct nouveau_clock *clk, struct nouveau_pstate *pstate)
+{
+        struct nouveau_clocks *clock = clk->domains - 1;
+        struct nouveau_cstate *cstate;
+        char info[3][32] = { "", "", "" };
+        char name[4] = "--";
+        int i = -1;
+
+        if (pstate->pstate != 0xff)
+                snprintf(name, sizeof(name), "%02x", pstate->pstate);
+
+        while ((++clock)->name != nv_clk_src_max) {
+                u32 lo = pstate->base.domain[clock->name];
+                u32 hi = lo;
+                if (hi == 0)
+                        continue;
+
+                nv_debug(clk, "%02x: %10d KHz\n", clock->name, lo);
+                list_for_each_entry(cstate, &pstate->list, head) {
+                        u32 freq = cstate->domain[clock->name];
+                        lo = min(lo, freq);
+                        hi = max(hi, freq);
+                        nv_debug(clk, "%10d KHz\n", freq);
+                }
+
+                if (clock->mname && ++i < ARRAY_SIZE(info)) {
+                        lo /= clock->mdiv;
+                        hi /= clock->mdiv;
+                        if (lo == hi) {
+                                snprintf(info[i], sizeof(info[i]), "%s %d MHz",
+                                         clock->mname, lo);
+                        } else {
+                                snprintf(info[i], sizeof(info[i]),
+                                         "%s %d-%d MHz", clock->mname, lo, hi);
+                        }
+                }
+        }
+
+        nv_info(clk, "%s: %s %s %s\n", name, info[0], info[1], info[2]);
+}
+
+static void
+nouveau_pstate_del(struct nouveau_pstate *pstate)
+{
+        struct nouveau_cstate *cstate, *temp;
+
+        list_for_each_entry_safe(cstate, temp, &pstate->list, head) {
+                nouveau_cstate_del(cstate);
+        }
+
+        list_del(&pstate->head);
+        kfree(pstate);
+}
+
+static int
+nouveau_pstate_new(struct nouveau_clock *clk, int idx)
+{
+        struct nouveau_bios *bios = nouveau_bios(clk);
+        struct nouveau_clocks *domain = clk->domains - 1;
+        struct nouveau_pstate *pstate;
+        struct nouveau_cstate *cstate;
+        struct nvbios_cstepE cstepE;
+        struct nvbios_perfE perfE;
+        u8  ver, hdr, cnt, len;
+        u16 data;
+
+        data = nvbios_perfEp(bios, idx, &ver, &hdr, &cnt, &len, &perfE);
+        if (!data)
+                return -EINVAL;
+        if (perfE.pstate == 0xff)
+                return 0;
+
+        pstate = kzalloc(sizeof(*pstate), GFP_KERNEL);
+        cstate = &pstate->base;
+        if (!pstate)
+                return -ENOMEM;
+
+        INIT_LIST_HEAD(&pstate->list);
+
+        pstate->pstate = perfE.pstate;
+        pstate->fanspeed = perfE.fanspeed;
+        cstate->voltage = perfE.voltage;
+        cstate->domain[nv_clk_src_core] = perfE.core;
+        cstate->domain[nv_clk_src_shader] = perfE.shader;
+        cstate->domain[nv_clk_src_mem] = perfE.memory;
+        cstate->domain[nv_clk_src_vdec] = perfE.vdec;
+        cstate->domain[nv_clk_src_dom6] = perfE.disp;
+
+        while (ver >= 0x40 && (++domain)->name != nv_clk_src_max) {
+                struct nvbios_perfS perfS;
+                u8  sver = ver, shdr = hdr;
+                u32 perfSe = nvbios_perfSp(bios, data, domain->bios,
+                                          &sver, &shdr, cnt, len, &perfS);
+                if (perfSe == 0 || sver != 0x40)
+                        continue;
+
+                if (domain->flags & NVKM_CLK_DOM_FLAG_CORE) {
+                        perfS.v40.freq = nouveau_clock_adjust(clk, false,
+                                                              pstate->pstate,
+                                                              domain->bios,
+                                                              perfS.v40.freq);
+                }
+
+                cstate->domain[domain->name] = perfS.v40.freq;
+        }
+
+        data = nvbios_cstepEm(bios, pstate->pstate, &ver, &hdr, &cstepE);
+        if (data) {
+                int idx = cstepE.index;
+                do {
+                        nouveau_cstate_new(clk, idx, pstate);
+                } while(idx--);
+        }
+
+        nouveau_pstate_info(clk, pstate);
+        list_add_tail(&pstate->head, &clk->states);
+        clk->state_nr++;
+        return 0;
+}
+
+/******************************************************************************
+ * Adjustment triggers
+ *****************************************************************************/
+static int
+nouveau_clock_ustate_update(struct nouveau_clock *clk, int req)
+{
+        struct nouveau_pstate *pstate;
+        int i = 0;
+
+        /* YKW repellant */
+        return -ENOSYS;
+
+        if (req != -1 && req != -2) {
+                list_for_each_entry(pstate, &clk->states, head) {
+                        if (pstate->pstate == req)
+                                break;
+                        i++;
+                }
+
+                if (pstate->pstate != req)
+                        return -EINVAL;
+                req = i;
+        }
+
+        clk->ustate = req;
+        return 0;
+}
+
+int
+nouveau_clock_ustate(struct nouveau_clock *clk, int req)
+{
+        int ret = nouveau_clock_ustate_update(clk, req);
+        if (ret)
+                return ret;
+        return nouveau_pstate_calc(clk);
+}
+
+int
+nouveau_clock_astate(struct nouveau_clock *clk, int req, int rel)
+{
+        if (!rel) clk->astate  = req;
+        if ( rel) clk->astate += rel;
+        clk->astate = min(clk->astate, clk->state_nr - 1);
+        clk->astate = max(clk->astate, 0);
+        return nouveau_pstate_calc(clk);
+}
+
+int
+nouveau_clock_tstate(struct nouveau_clock *clk, int req, int rel)
+{
+        if (!rel) clk->tstate  = req;
+        if ( rel) clk->tstate += rel;
+        clk->tstate = min(clk->tstate, 0);
+        clk->tstate = max(clk->tstate, -(clk->state_nr - 1));
+        return nouveau_pstate_calc(clk);
+}
+
+int
+nouveau_clock_dstate(struct nouveau_clock *clk, int req, int rel)
+{
+        if (!rel) clk->dstate  = req;
+        if ( rel) clk->dstate += rel;
+        clk->dstate = min(clk->dstate, clk->state_nr - 1);
+        clk->dstate = max(clk->dstate, 0);
+        return nouveau_pstate_calc(clk);
+}
+
+/******************************************************************************
+ * subdev base class implementation
+ *****************************************************************************/
+int
+_nouveau_clock_init(struct nouveau_object *object)
+{
+        struct nouveau_clock *clk = (void *)object;
+        struct nouveau_clocks *clock = clk->domains;
+        int ret;
+
+        memset(&clk->bstate, 0x00, sizeof(clk->bstate));
+        INIT_LIST_HEAD(&clk->bstate.list);
+        clk->bstate.pstate = 0xff;
+
+        while (clock->name != nv_clk_src_max) {
+                ret = clk->read(clk, clock->name);
+                if (ret < 0) {
+                        nv_error(clk, "%02x freq unknown\n", clock->name);
+                        return ret;
+                }
+                clk->bstate.base.domain[clock->name] = ret;
+                clock++;
+        }
+
+        nouveau_pstate_info(clk, &clk->bstate);
+
+        clk->astate = clk->state_nr - 1;
+        clk->tstate = 0;
+        clk->dstate = 0;
+        clk->pstate = -1;
+        nouveau_pstate_calc(clk);
+        return 0;
+}
+
+void
+_nouveau_clock_dtor(struct nouveau_object *object)
+{
+        struct nouveau_clock *clk = (void *)object;
+        struct nouveau_pstate *pstate, *temp;
+
+        list_for_each_entry_safe(pstate, temp, &clk->states, head) {
+                nouveau_pstate_del(pstate);
+        }
+
+        nouveau_subdev_destroy(&clk->base);
+}
+
+int
+nouveau_clock_create_(struct nouveau_object *parent,
+                      struct nouveau_object *engine,
+                      struct nouveau_oclass *oclass,
+                      struct nouveau_clocks *clocks,
+                      int length, void **object)
+{
+        struct nouveau_device *device = nv_device(parent);
+        struct nouveau_clock *clk;
+        int ret, idx, arglen;
+        const char *mode;
+
+        ret = nouveau_subdev_create_(parent, engine, oclass, 0, "CLK",
+                                     "clock", length, object);
+        clk = *object;
+        if (ret)
+                return ret;
+
+        INIT_LIST_HEAD(&clk->states);
+        clk->domains = clocks;
+        clk->ustate = -1;
+
+        idx = 0;
+        do {
+                ret = nouveau_pstate_new(clk, idx++);
+        } while (ret == 0);
+
+        mode = nouveau_stropt(device->cfgopt, "NvClkMode", &arglen);
+        if (mode) {
+                if (!strncasecmpz(mode, "disabled", arglen)) {
+                        clk->ustate = -1;
+                } else {
+                        char save = mode[arglen];
+                        long v;
+
+                        ((char *)mode)[arglen] = '\0';
+                        if (!kstrtol(mode, 0, &v))
+                                nouveau_clock_ustate_update(clk, v);
+                        ((char *)mode)[arglen] = save;
+                }
+        }
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nv04.c
index a14277586595..da50c1b12928 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nv04.c
@@ -77,7 +77,7 @@ nv04_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nv04_clock_priv *priv;
         int ret;
 
-        ret = nouveau_clock_create(parent, engine, oclass, &priv);
+        ret = nouveau_clock_create(parent, engine, oclass, NULL, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nv40.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nv40.c
index 0db5dbfd91b5..db7346f79080 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/nv40.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nv40.c
@@ -23,11 +23,188 @@
  */
 
 #include <subdev/clock.h>
+#include <subdev/bios.h>
+#include <subdev/bios/pll.h>
+
+#include "pll.h"
 
 struct nv40_clock_priv {
         struct nouveau_clock base;
+        u32 ctrl;
+        u32 npll_ctrl;
+        u32 npll_coef;
+        u32 spll;
+};
+
+static struct nouveau_clocks
+nv40_domain[] = {
+        { nv_clk_src_crystal, 0xff },
+        { nv_clk_src_href   , 0xff },
+        { nv_clk_src_core   , 0xff, 0, "core", 1000 },
+        { nv_clk_src_shader , 0xff, 0, "shader", 1000 },
+        { nv_clk_src_mem    , 0xff, 0, "memory", 1000 },
+        { nv_clk_src_max }
 };
 
+static u32
+read_pll_1(struct nv40_clock_priv *priv, u32 reg)
+{
+        u32 ctrl = nv_rd32(priv, reg + 0x00);
+        int P = (ctrl & 0x00070000) >> 16;
+        int N = (ctrl & 0x0000ff00) >> 8;
+        int M = (ctrl & 0x000000ff) >> 0;
+        u32 ref = 27000, clk = 0;
+
+        if (ctrl & 0x80000000)
+                clk = ref * N / M;
+
+        return clk >> P;
+}
+
+static u32
+read_pll_2(struct nv40_clock_priv *priv, u32 reg)
+{
+        u32 ctrl = nv_rd32(priv, reg + 0x00);
+        u32 coef = nv_rd32(priv, reg + 0x04);
+        int N2 = (coef & 0xff000000) >> 24;
+        int M2 = (coef & 0x00ff0000) >> 16;
+        int N1 = (coef & 0x0000ff00) >> 8;
+        int M1 = (coef & 0x000000ff) >> 0;
+        int P = (ctrl & 0x00070000) >> 16;
+        u32 ref = 27000, clk = 0;
+
+        if ((ctrl & 0x80000000) && M1) {
+                clk = ref * N1 / M1;
+                if ((ctrl & 0x40000100) == 0x40000000) {
+                        if (M2)
+                                clk = clk * N2 / M2;
+                        else
+                                clk = 0;
+                }
+        }
+
+        return clk >> P;
+}
+
+static u32
+read_clk(struct nv40_clock_priv *priv, u32 src)
+{
+        switch (src) {
+        case 3:
+                return read_pll_2(priv, 0x004000);
+        case 2:
+                return read_pll_1(priv, 0x004008);
+        default:
+                break;
+        }
+
+        return 0;
+}
+
+static int
+nv40_clock_read(struct nouveau_clock *clk, enum nv_clk_src src)
+{
+        struct nv40_clock_priv *priv = (void *)clk;
+        u32 mast = nv_rd32(priv, 0x00c040);
+
+        switch (src) {
+        case nv_clk_src_crystal:
+                return nv_device(priv)->crystal;
+        case nv_clk_src_href:
+                return 100000; /*XXX: PCIE/AGP differ*/
+        case nv_clk_src_core:
+                return read_clk(priv, (mast & 0x00000003) >> 0);
+        case nv_clk_src_shader:
+                return read_clk(priv, (mast & 0x00000030) >> 4);
+        case nv_clk_src_mem:
+                return read_pll_2(priv, 0x4020);
+        default:
+                break;
+        }
+
+        nv_debug(priv, "unknown clock source %d 0x%08x\n", src, mast);
+        return -EINVAL;
+}
+
+static int
+nv40_clock_calc_pll(struct nv40_clock_priv *priv, u32 reg, u32 clk,
+                    int *N1, int *M1, int *N2, int *M2, int *log2P)
+{
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        struct nvbios_pll pll;
+        int ret;
+
+        ret = nvbios_pll_parse(bios, reg, &pll);
+        if (ret)
+                return ret;
+
+        if (clk < pll.vco1.max_freq)
+                pll.vco2.max_freq = 0;
+
+        ret = nv04_pll_calc(nv_subdev(priv), &pll, clk, N1, M1, N2, M2, log2P);
+        if (ret == 0)
+                return -ERANGE;
+        return ret;
+}
+
+static int
+nv40_clock_calc(struct nouveau_clock *clk, struct nouveau_cstate *cstate)
+{
+        struct nv40_clock_priv *priv = (void *)clk;
+        int gclk = cstate->domain[nv_clk_src_core];
+        int sclk = cstate->domain[nv_clk_src_shader];
+        int N1, M1, N2, M2, log2P;
+        int ret;
+
+        /* core/geometric clock */
+        ret = nv40_clock_calc_pll(priv, 0x004000, gclk,
+                                 &N1, &M1, &N2, &M2, &log2P);
+        if (ret < 0)
+                return ret;
+
+        if (N2 == M2) {
+                priv->npll_ctrl = 0x80000100 | (log2P << 16);
+                priv->npll_coef = (N1 << 8) | M1;
+        } else {
+                priv->npll_ctrl = 0xc0000000 | (log2P << 16);
+                priv->npll_coef = (N2 << 24) | (M2 << 16) | (N1 << 8) | M1;
+        }
+
+        /* use the second pll for shader/rop clock, if it differs from core */
+        if (sclk && sclk != gclk) {
+                ret = nv40_clock_calc_pll(priv, 0x004008, sclk,
+                                         &N1, &M1, NULL, NULL, &log2P);
+                if (ret < 0)
+                        return ret;
+
+                priv->spll = 0xc0000000 | (log2P << 16) | (N1 << 8) | M1;
+                priv->ctrl = 0x00000223;
+        } else {
+                priv->spll = 0x00000000;
+                priv->ctrl = 0x00000333;
+        }
+
+        return 0;
+}
+
+static int
+nv40_clock_prog(struct nouveau_clock *clk)
+{
+        struct nv40_clock_priv *priv = (void *)clk;
+        nv_mask(priv, 0x00c040, 0x00000333, 0x00000000);
+        nv_wr32(priv, 0x004004, priv->npll_coef);
+        nv_mask(priv, 0x004000, 0xc0070100, priv->npll_ctrl);
+        nv_mask(priv, 0x004008, 0xc007ffff, priv->spll);
+        mdelay(5);
+        nv_mask(priv, 0x00c040, 0x00000333, priv->ctrl);
+        return 0;
+}
+
+static void
+nv40_clock_tidy(struct nouveau_clock *clk)
+{
+}
+
 static int
 nv40_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                 struct nouveau_oclass *oclass, void *data, u32 size,
@@ -36,13 +213,17 @@ nv40_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nv40_clock_priv *priv;
         int ret;
 
-        ret = nouveau_clock_create(parent, engine, oclass, &priv);
+        ret = nouveau_clock_create(parent, engine, oclass, nv40_domain, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
 
         priv->base.pll_calc = nv04_clock_pll_calc;
         priv->base.pll_prog = nv04_clock_pll_prog;
+        priv->base.read = nv40_clock_read;
+        priv->base.calc = nv40_clock_calc;
+        priv->base.prog = nv40_clock_prog;
+        priv->base.tidy = nv40_clock_tidy;
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nv50.c
index d09d3e78040c..250a6d96016b 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nv50.c
@@ -22,40 +22,538 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/clock.h>
 #include <subdev/bios.h>
 #include <subdev/bios/pll.h>
 
+#include "nv50.h"
 #include "pll.h"
+#include "seq.h"
 
-struct nv50_clock_priv {
-        struct nouveau_clock base;
-};
+static u32
+read_div(struct nv50_clock_priv *priv)
+{
+        switch (nv_device(priv)->chipset) {
+        case 0x50: /* it exists, but only has bit 31, not the dividers.. */
+        case 0x84:
+        case 0x86:
+        case 0x98:
+        case 0xa0:
+                return nv_rd32(priv, 0x004700);
+        case 0x92:
+        case 0x94:
+        case 0x96:
+                return nv_rd32(priv, 0x004800);
+        default:
+                return 0x00000000;
+        }
+}
+
+static u32
+read_pll_src(struct nv50_clock_priv *priv, u32 base)
+{
+        struct nouveau_clock *clk = &priv->base;
+        u32 coef, ref = clk->read(clk, nv_clk_src_crystal);
+        u32 rsel = nv_rd32(priv, 0x00e18c);
+        int P, N, M, id;
+
+        switch (nv_device(priv)->chipset) {
+        case 0x50:
+        case 0xa0:
+                switch (base) {
+                case 0x4020:
+                case 0x4028: id = !!(rsel & 0x00000004); break;
+                case 0x4008: id = !!(rsel & 0x00000008); break;
+                case 0x4030: id = 0; break;
+                default:
+                        nv_error(priv, "ref: bad pll 0x%06x\n", base);
+                        return 0;
+                }
+
+                coef = nv_rd32(priv, 0x00e81c + (id * 0x0c));
+                ref *=  (coef & 0x01000000) ? 2 : 4;
+                P    =  (coef & 0x00070000) >> 16;
+                N    = ((coef & 0x0000ff00) >> 8) + 1;
+                M    = ((coef & 0x000000ff) >> 0) + 1;
+                break;
+        case 0x84:
+        case 0x86:
+        case 0x92:
+                coef = nv_rd32(priv, 0x00e81c);
+                P    = (coef & 0x00070000) >> 16;
+                N    = (coef & 0x0000ff00) >> 8;
+                M    = (coef & 0x000000ff) >> 0;
+                break;
+        case 0x94:
+        case 0x96:
+        case 0x98:
+                rsel = nv_rd32(priv, 0x00c050);
+                switch (base) {
+                case 0x4020: rsel = (rsel & 0x00000003) >> 0; break;
+                case 0x4008: rsel = (rsel & 0x0000000c) >> 2; break;
+                case 0x4028: rsel = (rsel & 0x00001800) >> 11; break;
+                case 0x4030: rsel = 3; break;
+                default:
+                        nv_error(priv, "ref: bad pll 0x%06x\n", base);
+                        return 0;
+                }
+
+                switch (rsel) {
+                case 0: id = 1; break;
+                case 1: return clk->read(clk, nv_clk_src_crystal);
+                case 2: return clk->read(clk, nv_clk_src_href);
+                case 3: id = 0; break;
+                }
+
+                coef =  nv_rd32(priv, 0x00e81c + (id * 0x28));
+                P    = (nv_rd32(priv, 0x00e824 + (id * 0x28)) >> 16) & 7;
+                P   += (coef & 0x00070000) >> 16;
+                N    = (coef & 0x0000ff00) >> 8;
+                M    = (coef & 0x000000ff) >> 0;
+                break;
+        default:
+                BUG_ON(1);
+        }
+
+        if (M)
+                return (ref * N / M) >> P;
+        return 0;
+}
+
+static u32
+read_pll_ref(struct nv50_clock_priv *priv, u32 base)
+{
+        struct nouveau_clock *clk = &priv->base;
+        u32 src, mast = nv_rd32(priv, 0x00c040);
+
+        switch (base) {
+        case 0x004028:
+                src = !!(mast & 0x00200000);
+                break;
+        case 0x004020:
+                src = !!(mast & 0x00400000);
+                break;
+        case 0x004008:
+                src = !!(mast & 0x00010000);
+                break;
+        case 0x004030:
+                src = !!(mast & 0x02000000);
+                break;
+        case 0x00e810:
+                return clk->read(clk, nv_clk_src_crystal);
+        default:
+                nv_error(priv, "bad pll 0x%06x\n", base);
+                return 0;
+        }
+
+        if (src)
+                return clk->read(clk, nv_clk_src_href);
+        return read_pll_src(priv, base);
+}
+
+static u32
+read_pll(struct nv50_clock_priv *priv, u32 base)
+{
+        struct nouveau_clock *clk = &priv->base;
+        u32 mast = nv_rd32(priv, 0x00c040);
+        u32 ctrl = nv_rd32(priv, base + 0);
+        u32 coef = nv_rd32(priv, base + 4);
+        u32 ref = read_pll_ref(priv, base);
+        u32 freq = 0;
+        int N1, N2, M1, M2;
+
+        if (base == 0x004028 && (mast & 0x00100000)) {
+                /* wtf, appears to only disable post-divider on nva0 */
+                if (nv_device(priv)->chipset != 0xa0)
+                        return clk->read(clk, nv_clk_src_dom6);
+        }
+
+        N2 = (coef & 0xff000000) >> 24;
+        M2 = (coef & 0x00ff0000) >> 16;
+        N1 = (coef & 0x0000ff00) >> 8;
+        M1 = (coef & 0x000000ff);
+        if ((ctrl & 0x80000000) && M1) {
+                freq = ref * N1 / M1;
+                if ((ctrl & 0x40000100) == 0x40000000) {
+                        if (M2)
+                                freq = freq * N2 / M2;
+                        else
+                                freq = 0;
+                }
+        }
+
+        return freq;
+}
 
 static int
+nv50_clock_read(struct nouveau_clock *clk, enum nv_clk_src src)
+{
+        struct nv50_clock_priv *priv = (void *)clk;
+        u32 mast = nv_rd32(priv, 0x00c040);
+        u32 P = 0;
+
+        switch (src) {
+        case nv_clk_src_crystal:
+                return nv_device(priv)->crystal;
+        case nv_clk_src_href:
+                return 100000; /* PCIE reference clock */
+        case nv_clk_src_hclk:
+                return div_u64((u64)clk->read(clk, nv_clk_src_href) * 27778, 10000);
+        case nv_clk_src_hclkm3:
+                return clk->read(clk, nv_clk_src_hclk) * 3;
+        case nv_clk_src_hclkm3d2:
+                return clk->read(clk, nv_clk_src_hclk) * 3 / 2;
+        case nv_clk_src_host:
+                switch (mast & 0x30000000) {
+                case 0x00000000: return clk->read(clk, nv_clk_src_href);
+                case 0x10000000: break;
+                case 0x20000000: /* !0x50 */
+                case 0x30000000: return clk->read(clk, nv_clk_src_hclk);
+                }
+                break;
+        case nv_clk_src_core:
+                if (!(mast & 0x00100000))
+                        P = (nv_rd32(priv, 0x004028) & 0x00070000) >> 16;
+                switch (mast & 0x00000003) {
+                case 0x00000000: return clk->read(clk, nv_clk_src_crystal) >> P;
+                case 0x00000001: return clk->read(clk, nv_clk_src_dom6);
+                case 0x00000002: return read_pll(priv, 0x004020) >> P;
+                case 0x00000003: return read_pll(priv, 0x004028) >> P;
+                }
+                break;
+        case nv_clk_src_shader:
+                P = (nv_rd32(priv, 0x004020) & 0x00070000) >> 16;
+                switch (mast & 0x00000030) {
+                case 0x00000000:
+                        if (mast & 0x00000080)
+                                return clk->read(clk, nv_clk_src_host) >> P;
+                        return clk->read(clk, nv_clk_src_crystal) >> P;
+                case 0x00000010: break;
+                case 0x00000020: return read_pll(priv, 0x004028) >> P;
+                case 0x00000030: return read_pll(priv, 0x004020) >> P;
+                }
+                break;
+        case nv_clk_src_mem:
+                P = (nv_rd32(priv, 0x004008) & 0x00070000) >> 16;
+                if (nv_rd32(priv, 0x004008) & 0x00000200) {
+                        switch (mast & 0x0000c000) {
+                        case 0x00000000:
+                                return clk->read(clk, nv_clk_src_crystal) >> P;
+                        case 0x00008000:
+                        case 0x0000c000:
+                                return clk->read(clk, nv_clk_src_href) >> P;
+                        }
+                } else {
+                        return read_pll(priv, 0x004008) >> P;
+                }
+                break;
+        case nv_clk_src_vdec:
+                P = (read_div(priv) & 0x00000700) >> 8;
+                switch (nv_device(priv)->chipset) {
+                case 0x84:
+                case 0x86:
+                case 0x92:
+                case 0x94:
+                case 0x96:
+                case 0xa0:
+                        switch (mast & 0x00000c00) {
+                        case 0x00000000:
+                                if (nv_device(priv)->chipset == 0xa0) /* wtf?? */
+                                        return clk->read(clk, nv_clk_src_core) >> P;
+                                return clk->read(clk, nv_clk_src_crystal) >> P;
+                        case 0x00000400:
+                                return 0;
+                        case 0x00000800:
+                                if (mast & 0x01000000)
+                                        return read_pll(priv, 0x004028) >> P;
+                                return read_pll(priv, 0x004030) >> P;
+                        case 0x00000c00:
+                                return clk->read(clk, nv_clk_src_core) >> P;
+                        }
+                        break;
+                case 0x98:
+                        switch (mast & 0x00000c00) {
+                        case 0x00000000:
+                                return clk->read(clk, nv_clk_src_core) >> P;
+                        case 0x00000400:
+                                return 0;
+                        case 0x00000800:
+                                return clk->read(clk, nv_clk_src_hclkm3d2) >> P;
+                        case 0x00000c00:
+                                return clk->read(clk, nv_clk_src_mem) >> P;
+                        }
+                        break;
+                }
+                break;
+        case nv_clk_src_dom6:
+                switch (nv_device(priv)->chipset) {
+                case 0x50:
+                case 0xa0:
+                        return read_pll(priv, 0x00e810) >> 2;
+                case 0x84:
+                case 0x86:
+                case 0x92:
+                case 0x94:
+                case 0x96:
+                case 0x98:
+                        P = (read_div(priv) & 0x00000007) >> 0;
+                        switch (mast & 0x0c000000) {
+                        case 0x00000000: return clk->read(clk, nv_clk_src_href);
+                        case 0x04000000: break;
+                        case 0x08000000: return clk->read(clk, nv_clk_src_hclk);
+                        case 0x0c000000:
+                                return clk->read(clk, nv_clk_src_hclkm3) >> P;
+                        }
+                        break;
+                default:
+                        break;
+                }
+        default:
+                break;
+        }
+
+        nv_debug(priv, "unknown clock source %d 0x%08x\n", src, mast);
+        return -EINVAL;
+}
+
+static u32
+calc_pll(struct nv50_clock_priv *priv, u32 reg, u32 clk, int *N, int *M, int *P)
+{
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        struct nvbios_pll pll;
+        int ret;
+
+        ret = nvbios_pll_parse(bios, reg, &pll);
+        if (ret)
+                return 0;
+
+        pll.vco2.max_freq = 0;
+        pll.refclk = read_pll_ref(priv, reg);
+        if (!pll.refclk)
+                return 0;
+
+        return nv04_pll_calc(nv_subdev(priv), &pll, clk, N, M, NULL, NULL, P);
+}
+
+static inline u32
+calc_div(u32 src, u32 target, int *div)
+{
+        u32 clk0 = src, clk1 = src;
+        for (*div = 0; *div <= 7; (*div)++) {
+                if (clk0 <= target) {
+                        clk1 = clk0 << (*div ? 1 : 0);
+                        break;
+                }
+                clk0 >>= 1;
+        }
+
+        if (target - clk0 <= clk1 - target)
+                return clk0;
+        (*div)--;
+        return clk1;
+}
+
+static inline u32
+clk_same(u32 a, u32 b)
+{
+        return ((a / 1000) == (b / 1000));
+}
+
+static int
+nv50_clock_calc(struct nouveau_clock *clk, struct nouveau_cstate *cstate)
+{
+        struct nv50_clock_priv *priv = (void *)clk;
+        struct nv50_clock_hwsq *hwsq = &priv->hwsq;
+        const int shader = cstate->domain[nv_clk_src_shader];
+        const int core = cstate->domain[nv_clk_src_core];
+        const int vdec = cstate->domain[nv_clk_src_vdec];
+        const int dom6 = cstate->domain[nv_clk_src_dom6];
+        u32 mastm = 0, mastv = 0;
+        u32 divsm = 0, divsv = 0;
+        int N, M, P1, P2;
+        int freq, out;
+
+        /* prepare a hwsq script from which we'll perform the reclock */
+        out = clk_init(hwsq, nv_subdev(clk));
+        if (out)
+                return out;
+
+        clk_wr32(hwsq, fifo, 0x00000001); /* block fifo */
+        clk_nsec(hwsq, 8000);
+        clk_setf(hwsq, 0x10, 0x00); /* disable fb */
+        clk_wait(hwsq, 0x00, 0x01); /* wait for fb disabled */
+
+        /* vdec: avoid modifying xpll until we know exactly how the other
+         * clock domains work, i suspect at least some of them can also be
+         * tied to xpll...
+         */
+        if (vdec) {
+                /* see how close we can get using nvclk as a source */
+                freq = calc_div(core, vdec, &P1);
+
+                /* see how close we can get using xpll/hclk as a source */
+                if (nv_device(priv)->chipset != 0x98)
+                        out = read_pll(priv, 0x004030);
+                else
+                        out = clk->read(clk, nv_clk_src_hclkm3d2);
+                out = calc_div(out, vdec, &P2);
+
+                /* select whichever gets us closest */
+                if (abs(vdec - freq) <= abs(vdec - out)) {
+                        if (nv_device(priv)->chipset != 0x98)
+                                mastv |= 0x00000c00;
+                        divsv |= P1 << 8;
+                } else {
+                        mastv |= 0x00000800;
+                        divsv |= P2 << 8;
+                }
+
+                mastm |= 0x00000c00;
+                divsm |= 0x00000700;
+        }
+
+        /* dom6: nfi what this is, but we're limited to various combinations
+         * of the host clock frequency
+         */
+        if (dom6) {
+                if (clk_same(dom6, clk->read(clk, nv_clk_src_href))) {
+                        mastv |= 0x00000000;
+                } else
+                if (clk_same(dom6, clk->read(clk, nv_clk_src_hclk))) {
+                        mastv |= 0x08000000;
+                } else {
+                        freq = clk->read(clk, nv_clk_src_hclk) * 3;
+                        freq = calc_div(freq, dom6, &P1);
+
+                        mastv |= 0x0c000000;
+                        divsv |= P1;
+                }
+
+                mastm |= 0x0c000000;
+                divsm |= 0x00000007;
+        }
+
+        /* vdec/dom6: switch to "safe" clocks temporarily, update dividers
+         * and then switch to target clocks
+         */
+        clk_mask(hwsq, mast, mastm, 0x00000000);
+        clk_mask(hwsq, divs, divsm, divsv);
+        clk_mask(hwsq, mast, mastm, mastv);
+
+        /* core/shader: disconnect nvclk/sclk from their PLLs (nvclk to dom6,
+         * sclk to hclk) before reprogramming
+         */
+        if (nv_device(priv)->chipset < 0x92)
+                clk_mask(hwsq, mast, 0x001000b0, 0x00100080);
+        else
+                clk_mask(hwsq, mast, 0x000000b3, 0x00000081);
+
+        /* core: for the moment at least, always use nvpll */
+        freq = calc_pll(priv, 0x4028, core, &N, &M, &P1);
+        if (freq == 0)
+                return -ERANGE;
+
+        clk_mask(hwsq, nvpll[0], 0xc03f0100,
+                                 0x80000000 | (P1 << 19) | (P1 << 16));
+        clk_mask(hwsq, nvpll[1], 0x0000ffff, (N << 8) | M);
+
+        /* shader: tie to nvclk if possible, otherwise use spll.  have to be
+         * very careful that the shader clock is at least twice the core, or
+         * some chipsets will be very unhappy.  i expect most or all of these
+         * cases will be handled by tying to nvclk, but it's possible there's
+         * corners
+         */
+        if (P1-- && shader == (core << 1)) {
+                clk_mask(hwsq, spll[0], 0xc03f0100, (P1 << 19) | (P1 << 16));
+                clk_mask(hwsq, mast, 0x00100033, 0x00000023);
+        } else {
+                freq = calc_pll(priv, 0x4020, shader, &N, &M, &P1);
+                if (freq == 0)
+                        return -ERANGE;
+
+                clk_mask(hwsq, spll[0], 0xc03f0100,
+                                        0x80000000 | (P1 << 19) | (P1 << 16));
+                clk_mask(hwsq, spll[1], 0x0000ffff, (N << 8) | M);
+                clk_mask(hwsq, mast, 0x00100033, 0x00000033);
+        }
+
+        /* restore normal operation */
+        clk_setf(hwsq, 0x10, 0x01); /* enable fb */
+        clk_wait(hwsq, 0x00, 0x00); /* wait for fb enabled */
+        clk_wr32(hwsq, fifo, 0x00000000); /* un-block fifo */
+        return 0;
+}
+
+static int
+nv50_clock_prog(struct nouveau_clock *clk)
+{
+        struct nv50_clock_priv *priv = (void *)clk;
+        return clk_exec(&priv->hwsq, true);
+}
+
+static void
+nv50_clock_tidy(struct nouveau_clock *clk)
+{
+        struct nv50_clock_priv *priv = (void *)clk;
+        clk_exec(&priv->hwsq, false);
+}
+
+int
 nv50_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                 struct nouveau_oclass *oclass, void *data, u32 size,
                 struct nouveau_object **pobject)
 {
+        struct nv50_clock_oclass *pclass = (void *)oclass;
         struct nv50_clock_priv *priv;
         int ret;
 
-        ret = nouveau_clock_create(parent, engine, oclass, &priv);
+        ret = nouveau_clock_create(parent, engine, oclass, pclass->domains,
+                                  &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
 
-        priv->base.pll_calc = nv04_clock_pll_calc;
+        priv->hwsq.r_fifo = hwsq_reg(0x002504);
+        priv->hwsq.r_spll[0] = hwsq_reg(0x004020);
+        priv->hwsq.r_spll[1] = hwsq_reg(0x004024);
+        priv->hwsq.r_nvpll[0] = hwsq_reg(0x004028);
+        priv->hwsq.r_nvpll[1] = hwsq_reg(0x00402c);
+        switch (nv_device(priv)->chipset) {
+        case 0x92:
+        case 0x94:
+        case 0x96:
+                priv->hwsq.r_divs = hwsq_reg(0x004800);
+                break;
+        default:
+                priv->hwsq.r_divs = hwsq_reg(0x004700);
+                break;
+        }
+        priv->hwsq.r_mast = hwsq_reg(0x00c040);
+
+        priv->base.read = nv50_clock_read;
+        priv->base.calc = nv50_clock_calc;
+        priv->base.prog = nv50_clock_prog;
+        priv->base.tidy = nv50_clock_tidy;
         return 0;
 }
 
-struct nouveau_oclass
-nv50_clock_oclass = {
-        .handle = NV_SUBDEV(CLOCK, 0x50),
-        .ofuncs = &(struct nouveau_ofuncs) {
+static struct nouveau_clocks
+nv50_domains[] = {
+        { nv_clk_src_crystal, 0xff },
+        { nv_clk_src_href   , 0xff },
+        { nv_clk_src_core   , 0xff, 0, "core", 1000 },
+        { nv_clk_src_shader , 0xff, 0, "shader", 1000 },
+        { nv_clk_src_mem    , 0xff, 0, "memory", 1000 },
+        { nv_clk_src_max }
+};
+
+struct nouveau_oclass *
+nv50_clock_oclass = &(struct nv50_clock_oclass) {
+        .base.handle = NV_SUBDEV(CLOCK, 0x50),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv50_clock_ctor,
                 .dtor = _nouveau_clock_dtor,
                 .init = _nouveau_clock_init,
                 .fini = _nouveau_clock_fini,
         },
-};
+        .domains = nv50_domains,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nv50.h b/drivers/gpu/drm/nouveau/core/subdev/clock/nv50.h
new file mode 100644
index 000000000000..f10917d789e8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nv50.h
@@ -0,0 +1,31 @@
+#ifndef __NVKM_CLK_NV50_H__
+#define __NVKM_CLK_NV50_H__
+
+#include <subdev/bus.h>
+#include <subdev/bus/hwsq.h>
+#include <subdev/clock.h>
+
+struct nv50_clock_hwsq {
+        struct hwsq base;
+        struct hwsq_reg r_fifo;
+        struct hwsq_reg r_spll[2];
+        struct hwsq_reg r_nvpll[2];
+        struct hwsq_reg r_divs;
+        struct hwsq_reg r_mast;
+};
+
+struct nv50_clock_priv {
+        struct nouveau_clock base;
+        struct nv50_clock_hwsq hwsq;
+};
+
+int  nv50_clock_ctor(struct nouveau_object *, struct nouveau_object *,
+                     struct nouveau_oclass *, void *, u32,
+                     struct nouveau_object **);
+
+struct nv50_clock_oclass {
+        struct nouveau_oclass base;
+        struct nouveau_clocks *domains;
+};
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nv84.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nv84.c
new file mode 100644
index 000000000000..b0b7c1437f10
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nv84.c
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include "nv50.h"
+
+static struct nouveau_clocks
+nv84_domains[] = {
+        { nv_clk_src_crystal, 0xff },
+        { nv_clk_src_href   , 0xff },
+        { nv_clk_src_core   , 0xff, 0, "core", 1000 },
+        { nv_clk_src_shader , 0xff, 0, "shader", 1000 },
+        { nv_clk_src_mem    , 0xff, 0, "memory", 1000 },
+        { nv_clk_src_vdec   , 0xff },
+        { nv_clk_src_max }
+};
+
+struct nouveau_oclass *
+nv84_clock_oclass = &(struct nv50_clock_oclass) {
+        .base.handle = NV_SUBDEV(CLOCK, 0x84),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv50_clock_ctor,
+                .dtor = _nouveau_clock_dtor,
+                .init = _nouveau_clock_init,
+                .fini = _nouveau_clock_fini,
+        },
+        .domains = nv84_domains,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c
index f074cd20bc9c..4f5a1373f002 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.c
@@ -22,33 +22,277 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/clock.h>
 #include <subdev/bios.h>
 #include <subdev/bios/pll.h>
+#include <subdev/timer.h>
 
 #include "pll.h"
 
+#include "nva3.h"
+
 struct nva3_clock_priv {
         struct nouveau_clock base;
+        struct nva3_clock_info eng[nv_clk_src_max];
 };
 
+static u32 read_clk(struct nva3_clock_priv *, int, bool);
+static u32 read_pll(struct nva3_clock_priv *, int, u32);
+
+static u32
+read_vco(struct nva3_clock_priv *priv, int clk)
+{
+        u32 sctl = nv_rd32(priv, 0x4120 + (clk * 4));
+        if ((sctl & 0x00000030) != 0x00000030)
+                return read_pll(priv, 0x41, 0x00e820);
+        return read_pll(priv, 0x42, 0x00e8a0);
+}
+
+static u32
+read_clk(struct nva3_clock_priv *priv, int clk, bool ignore_en)
+{
+        u32 sctl, sdiv, sclk;
+
+        /* refclk for the 0xe8xx plls is a fixed frequency */
+        if (clk >= 0x40) {
+                if (nv_device(priv)->chipset == 0xaf) {
+                        /* no joke.. seriously.. sigh.. */
+                        return nv_rd32(priv, 0x00471c) * 1000;
+                }
+
+                return nv_device(priv)->crystal;
+        }
+
+        sctl = nv_rd32(priv, 0x4120 + (clk * 4));
+        if (!ignore_en && !(sctl & 0x00000100))
+                return 0;
+
+        switch (sctl & 0x00003000) {
+        case 0x00000000:
+                return nv_device(priv)->crystal;
+        case 0x00002000:
+                if (sctl & 0x00000040)
+                        return 108000;
+                return 100000;
+        case 0x00003000:
+                sclk = read_vco(priv, clk);
+                sdiv = ((sctl & 0x003f0000) >> 16) + 2;
+                return (sclk * 2) / sdiv;
+        default:
+                return 0;
+        }
+}
+
+static u32
+read_pll(struct nva3_clock_priv *priv, int clk, u32 pll)
+{
+        u32 ctrl = nv_rd32(priv, pll + 0);
+        u32 sclk = 0, P = 1, N = 1, M = 1;
+
+        if (!(ctrl & 0x00000008)) {
+                if (ctrl & 0x00000001) {
+                        u32 coef = nv_rd32(priv, pll + 4);
+                        M = (coef & 0x000000ff) >> 0;
+                        N = (coef & 0x0000ff00) >> 8;
+                        P = (coef & 0x003f0000) >> 16;
+
+                        /* no post-divider on these.. */
+                        if ((pll & 0x00ff00) == 0x00e800)
+                                P = 1;
+
+                        sclk = read_clk(priv, 0x00 + clk, false);
+                }
+        } else {
+                sclk = read_clk(priv, 0x10 + clk, false);
+        }
+
+        if (M * P)
+                return sclk * N / (M * P);
+        return 0;
+}
+
+static int
+nva3_clock_read(struct nouveau_clock *clk, enum nv_clk_src src)
+{
+        struct nva3_clock_priv *priv = (void *)clk;
+
+        switch (src) {
+        case nv_clk_src_crystal:
+                return nv_device(priv)->crystal;
+        case nv_clk_src_href:
+                return 100000;
+        case nv_clk_src_core:
+                return read_pll(priv, 0x00, 0x4200);
+        case nv_clk_src_shader:
+                return read_pll(priv, 0x01, 0x4220);
+        case nv_clk_src_mem:
+                return read_pll(priv, 0x02, 0x4000);
+        case nv_clk_src_disp:
+                return read_clk(priv, 0x20, false);
+        case nv_clk_src_vdec:
+                return read_clk(priv, 0x21, false);
+        case nv_clk_src_daemon:
+                return read_clk(priv, 0x25, false);
+        default:
+                nv_error(clk, "invalid clock source %d\n", src);
+                return -EINVAL;
+        }
+}
+
 int
-nva3_clock_pll_calc(struct nouveau_clock *clock, struct nvbios_pll *info,
-                    int clk, struct nouveau_pll_vals *pv)
+nva3_clock_info(struct nouveau_clock *clock, int clk, u32 pll, u32 khz,
+                struct nva3_clock_info *info)
 {
-        int ret, N, M, P;
+        struct nouveau_bios *bios = nouveau_bios(clock);
+        struct nva3_clock_priv *priv = (void *)clock;
+        struct nvbios_pll limits;
+        u32 oclk, sclk, sdiv;
+        int P, N, M, diff;
+        int ret;
+
+        info->pll = 0;
+        info->clk = 0;
+
+        switch (khz) {
+        case 27000:
+                info->clk = 0x00000100;
+                return khz;
+        case 100000:
+                info->clk = 0x00002100;
+                return khz;
+        case 108000:
+                info->clk = 0x00002140;
+                return khz;
+        default:
+                sclk = read_vco(priv, clk);
+                sdiv = min((sclk * 2) / (khz - 2999), (u32)65);
+                /* if the clock has a PLL attached, and we can get a within
+                 * [-2, 3) MHz of a divider, we'll disable the PLL and use
+                 * the divider instead.
+                 *
+                 * divider can go as low as 2, limited here because NVIDIA
+                 * and the VBIOS on my NVA8 seem to prefer using the PLL
+                 * for 810MHz - is there a good reason?
+                 */
+                if (sdiv > 4) {
+                        oclk = (sclk * 2) / sdiv;
+                        diff = khz - oclk;
+                        if (!pll || (diff >= -2000 && diff < 3000)) {
+                                info->clk = (((sdiv - 2) << 16) | 0x00003100);
+                                return oclk;
+                        }
+                }
+
+                if (!pll)
+                        return -ERANGE;
+                break;
+        }
 
-        ret = nva3_pll_calc(nv_subdev(clock), info, clk, &N, NULL, &M, &P);
+        ret = nvbios_pll_parse(bios, pll, &limits);
+        if (ret)
+                return ret;
+
+        limits.refclk = read_clk(priv, clk - 0x10, true);
+        if (!limits.refclk)
+                return -EINVAL;
 
-        if (ret > 0) {
-                pv->refclk = info->refclk;
-                pv->N1 = N;
-                pv->M1 = M;
-                pv->log2P = P;
+        ret = nva3_pll_calc(nv_subdev(priv), &limits, khz, &N, NULL, &M, &P);
+        if (ret >= 0) {
+                info->clk = nv_rd32(priv, 0x4120 + (clk * 4));
+                info->pll = (P << 16) | (N << 8) | M;
         }
+
+        return ret ? ret : -ERANGE;
+}
+
+static int
+calc_clk(struct nva3_clock_priv *priv, struct nouveau_cstate *cstate,
+         int clk, u32 pll, int idx)
+{
+        int ret = nva3_clock_info(&priv->base, clk, pll, cstate->domain[idx],
+                                  &priv->eng[idx]);
+        if (ret >= 0)
+                return 0;
         return ret;
 }
 
+static void
+prog_pll(struct nva3_clock_priv *priv, int clk, u32 pll, int idx)
+{
+        struct nva3_clock_info *info = &priv->eng[idx];
+        const u32 src0 = 0x004120 + (clk * 4);
+        const u32 src1 = 0x004160 + (clk * 4);
+        const u32 ctrl = pll + 0;
+        const u32 coef = pll + 4;
+
+        if (info->pll) {
+                nv_mask(priv, src0, 0x00000101, 0x00000101);
+                nv_wr32(priv, coef, info->pll);
+                nv_mask(priv, ctrl, 0x00000015, 0x00000015);
+                nv_mask(priv, ctrl, 0x00000010, 0x00000000);
+                nv_wait(priv, ctrl, 0x00020000, 0x00020000);
+                nv_mask(priv, ctrl, 0x00000010, 0x00000010);
+                nv_mask(priv, ctrl, 0x00000008, 0x00000000);
+                nv_mask(priv, src1, 0x00000100, 0x00000000);
+                nv_mask(priv, src1, 0x00000001, 0x00000000);
+        } else {
+                nv_mask(priv, src1, 0x003f3141, 0x00000101 | info->clk);
+                nv_mask(priv, ctrl, 0x00000018, 0x00000018);
+                udelay(20);
+                nv_mask(priv, ctrl, 0x00000001, 0x00000000);
+                nv_mask(priv, src0, 0x00000100, 0x00000000);
+                nv_mask(priv, src0, 0x00000001, 0x00000000);
+        }
+}
+
+static void
+prog_clk(struct nva3_clock_priv *priv, int clk, int idx)
+{
+        struct nva3_clock_info *info = &priv->eng[idx];
+        nv_mask(priv, 0x004120 + (clk * 4), 0x003f3141, 0x00000101 | info->clk);
+}
+
+static int
+nva3_clock_calc(struct nouveau_clock *clk, struct nouveau_cstate *cstate)
+{
+        struct nva3_clock_priv *priv = (void *)clk;
+        int ret;
+
+        if ((ret = calc_clk(priv, cstate, 0x10, 0x4200, nv_clk_src_core)) ||
+            (ret = calc_clk(priv, cstate, 0x11, 0x4220, nv_clk_src_shader)) ||
+            (ret = calc_clk(priv, cstate, 0x20, 0x0000, nv_clk_src_disp)) ||
+            (ret = calc_clk(priv, cstate, 0x21, 0x0000, nv_clk_src_vdec)))
+                return ret;
+
+        return 0;
+}
+
+static int
+nva3_clock_prog(struct nouveau_clock *clk)
+{
+        struct nva3_clock_priv *priv = (void *)clk;
+        prog_pll(priv, 0x00, 0x004200, nv_clk_src_core);
+        prog_pll(priv, 0x01, 0x004220, nv_clk_src_shader);
+        prog_clk(priv, 0x20, nv_clk_src_disp);
+        prog_clk(priv, 0x21, nv_clk_src_vdec);
+        return 0;
+}
+
+static void
+nva3_clock_tidy(struct nouveau_clock *clk)
+{
+}
+
+static struct nouveau_clocks
+nva3_domain[] = {
+        { nv_clk_src_crystal, 0xff },
+        { nv_clk_src_href   , 0xff },
+        { nv_clk_src_core   , 0x00, 0, "core", 1000 },
+        { nv_clk_src_shader , 0x01, 0, "shader", 1000 },
+        { nv_clk_src_mem    , 0x02, 0, "memory", 1000 },
+        { nv_clk_src_vdec   , 0x03 },
+        { nv_clk_src_disp   , 0x04 },
+        { nv_clk_src_max }
+};
 
 static int
 nva3_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
@@ -58,12 +302,15 @@ nva3_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nva3_clock_priv *priv;
         int ret;
 
-        ret = nouveau_clock_create(parent, engine, oclass, &priv);
+        ret = nouveau_clock_create(parent, engine, oclass, nva3_domain, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
 
-        priv->base.pll_calc = nva3_clock_pll_calc;
+        priv->base.read = nva3_clock_read;
+        priv->base.calc = nva3_clock_calc;
+        priv->base.prog = nva3_clock_prog;
+        priv->base.tidy = nva3_clock_tidy;
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.h b/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.h
new file mode 100644
index 000000000000..6229a509b42e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nva3.h
@@ -0,0 +1,14 @@
+#ifndef __NVKM_CLK_NVA3_H__
+#define __NVKM_CLK_NVA3_H__
+
+#include <subdev/clock.h>
+
+struct nva3_clock_info {
+        u32 clk;
+        u32 pll;
+};
+
+int nva3_clock_info(struct nouveau_clock *, int, u32, u32,
+                    struct nva3_clock_info *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nvc0.c
index 439d81c26130..c3105720ed24 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nvc0.c
@@ -25,11 +25,408 @@
 #include <subdev/clock.h>
 #include <subdev/bios.h>
 #include <subdev/bios/pll.h>
+#include <subdev/timer.h>
 
 #include "pll.h"
 
+struct nvc0_clock_info {
+        u32 freq;
+        u32 ssel;
+        u32 mdiv;
+        u32 dsrc;
+        u32 ddiv;
+        u32 coef;
+};
+
 struct nvc0_clock_priv {
         struct nouveau_clock base;
+        struct nvc0_clock_info eng[16];
+};
+
+static u32 read_div(struct nvc0_clock_priv *, int, u32, u32);
+
+static u32
+read_vco(struct nvc0_clock_priv *priv, u32 dsrc)
+{
+        struct nouveau_clock *clk = &priv->base;
+        u32 ssrc = nv_rd32(priv, dsrc);
+        if (!(ssrc & 0x00000100))
+                return clk->read(clk, nv_clk_src_sppll0);
+        return clk->read(clk, nv_clk_src_sppll1);
+}
+
+static u32
+read_pll(struct nvc0_clock_priv *priv, u32 pll)
+{
+        struct nouveau_clock *clk = &priv->base;
+        u32 ctrl = nv_rd32(priv, pll + 0x00);
+        u32 coef = nv_rd32(priv, pll + 0x04);
+        u32 P = (coef & 0x003f0000) >> 16;
+        u32 N = (coef & 0x0000ff00) >> 8;
+        u32 M = (coef & 0x000000ff) >> 0;
+        u32 sclk;
+
+        if (!(ctrl & 0x00000001))
+                return 0;
+
+        switch (pll) {
+        case 0x00e800:
+        case 0x00e820:
+                sclk = nv_device(priv)->crystal;
+                P = 1;
+                break;
+        case 0x132000:
+                sclk = clk->read(clk, nv_clk_src_mpllsrc);
+                break;
+        case 0x132020:
+                sclk = clk->read(clk, nv_clk_src_mpllsrcref);
+                break;
+        case 0x137000:
+        case 0x137020:
+        case 0x137040:
+        case 0x1370e0:
+                sclk = read_div(priv, (pll & 0xff) / 0x20, 0x137120, 0x137140);
+                break;
+        default:
+                return 0;
+        }
+
+        return sclk * N / M / P;
+}
+
+static u32
+read_div(struct nvc0_clock_priv *priv, int doff, u32 dsrc, u32 dctl)
+{
+        u32 ssrc = nv_rd32(priv, dsrc + (doff * 4));
+        u32 sctl = nv_rd32(priv, dctl + (doff * 4));
+
+        switch (ssrc & 0x00000003) {
+        case 0:
+                if ((ssrc & 0x00030000) != 0x00030000)
+                        return nv_device(priv)->crystal;
+                return 108000;
+        case 2:
+                return 100000;
+        case 3:
+                if (sctl & 0x80000000) {
+                        u32 sclk = read_vco(priv, dsrc + (doff * 4));
+                        u32 sdiv = (sctl & 0x0000003f) + 2;
+                        return (sclk * 2) / sdiv;
+                }
+
+                return read_vco(priv, dsrc + (doff * 4));
+        default:
+                return 0;
+        }
+}
+
+static u32
+read_clk(struct nvc0_clock_priv *priv, int clk)
+{
+        u32 sctl = nv_rd32(priv, 0x137250 + (clk * 4));
+        u32 ssel = nv_rd32(priv, 0x137100);
+        u32 sclk, sdiv;
+
+        if (ssel & (1 << clk)) {
+                if (clk < 7)
+                        sclk = read_pll(priv, 0x137000 + (clk * 0x20));
+                else
+                        sclk = read_pll(priv, 0x1370e0);
+                sdiv = ((sctl & 0x00003f00) >> 8) + 2;
+        } else {
+                sclk = read_div(priv, clk, 0x137160, 0x1371d0);
+                sdiv = ((sctl & 0x0000003f) >> 0) + 2;
+        }
+
+        if (sctl & 0x80000000)
+                return (sclk * 2) / sdiv;
+
+        return sclk;
+}
+
+static int
+nvc0_clock_read(struct nouveau_clock *clk, enum nv_clk_src src)
+{
+        struct nouveau_device *device = nv_device(clk);
+        struct nvc0_clock_priv *priv = (void *)clk;
+
+        switch (src) {
+        case nv_clk_src_crystal:
+                return device->crystal;
+        case nv_clk_src_href:
+                return 100000;
+        case nv_clk_src_sppll0:
+                return read_pll(priv, 0x00e800);
+        case nv_clk_src_sppll1:
+                return read_pll(priv, 0x00e820);
+
+        case nv_clk_src_mpllsrcref:
+                return read_div(priv, 0, 0x137320, 0x137330);
+        case nv_clk_src_mpllsrc:
+                return read_pll(priv, 0x132020);
+        case nv_clk_src_mpll:
+                return read_pll(priv, 0x132000);
+        case nv_clk_src_mdiv:
+                return read_div(priv, 0, 0x137300, 0x137310);
+        case nv_clk_src_mem:
+                if (nv_rd32(priv, 0x1373f0) & 0x00000002)
+                        return clk->read(clk, nv_clk_src_mpll);
+                return clk->read(clk, nv_clk_src_mdiv);
+
+        case nv_clk_src_gpc:
+                return read_clk(priv, 0x00);
+        case nv_clk_src_rop:
+                return read_clk(priv, 0x01);
+        case nv_clk_src_hubk07:
+                return read_clk(priv, 0x02);
+        case nv_clk_src_hubk06:
+                return read_clk(priv, 0x07);
+        case nv_clk_src_hubk01:
+                return read_clk(priv, 0x08);
+        case nv_clk_src_copy:
+                return read_clk(priv, 0x09);
+        case nv_clk_src_daemon:
+                return read_clk(priv, 0x0c);
+        case nv_clk_src_vdec:
+                return read_clk(priv, 0x0e);
+        default:
+                nv_error(clk, "invalid clock source %d\n", src);
+                return -EINVAL;
+        }
+}
+
+static u32
+calc_div(struct nvc0_clock_priv *priv, int clk, u32 ref, u32 freq, u32 *ddiv)
+{
+        u32 div = min((ref * 2) / freq, (u32)65);
+        if (div < 2)
+                div = 2;
+
+        *ddiv = div - 2;
+        return (ref * 2) / div;
+}
+
+static u32
+calc_src(struct nvc0_clock_priv *priv, int clk, u32 freq, u32 *dsrc, u32 *ddiv)
+{
+        u32 sclk;
+
+        /* use one of the fixed frequencies if possible */
+        *ddiv = 0x00000000;
+        switch (freq) {
+        case  27000:
+        case 108000:
+                *dsrc = 0x00000000;
+                if (freq == 108000)
+                        *dsrc |= 0x00030000;
+                return freq;
+        case 100000:
+                *dsrc = 0x00000002;
+                return freq;
+        default:
+                *dsrc = 0x00000003;
+                break;
+        }
+
+        /* otherwise, calculate the closest divider */
+        sclk = read_vco(priv, 0x137160 + (clk * 4));
+        if (clk < 7)
+                sclk = calc_div(priv, clk, sclk, freq, ddiv);
+        return sclk;
+}
+
+static u32
+calc_pll(struct nvc0_clock_priv *priv, int clk, u32 freq, u32 *coef)
+{
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        struct nvbios_pll limits;
+        int N, M, P, ret;
+
+        ret = nvbios_pll_parse(bios, 0x137000 + (clk * 0x20), &limits);
+        if (ret)
+                return 0;
+
+        limits.refclk = read_div(priv, clk, 0x137120, 0x137140);
+        if (!limits.refclk)
+                return 0;
+
+        ret = nva3_pll_calc(nv_subdev(priv), &limits, freq, &N, NULL, &M, &P);
+        if (ret <= 0)
+                return 0;
+
+        *coef = (P << 16) | (N << 8) | M;
+        return ret;
+}
+
+static int
+calc_clk(struct nvc0_clock_priv *priv,
+         struct nouveau_cstate *cstate, int clk, int dom)
+{
+        struct nvc0_clock_info *info = &priv->eng[clk];
+        u32 freq = cstate->domain[dom];
+        u32 src0, div0, div1D, div1P = 0;
+        u32 clk0, clk1 = 0;
+
+        /* invalid clock domain */
+        if (!freq)
+                return 0;
+
+        /* first possible path, using only dividers */
+        clk0 = calc_src(priv, clk, freq, &src0, &div0);
+        clk0 = calc_div(priv, clk, clk0, freq, &div1D);
+
+        /* see if we can get any closer using PLLs */
+        if (clk0 != freq && (0x00004387 & (1 << clk))) {
+                if (clk <= 7)
+                        clk1 = calc_pll(priv, clk, freq, &info->coef);
+                else
+                        clk1 = cstate->domain[nv_clk_src_hubk06];
+                clk1 = calc_div(priv, clk, clk1, freq, &div1P);
+        }
+
+        /* select the method which gets closest to target freq */
+        if (abs((int)freq - clk0) <= abs((int)freq - clk1)) {
+                info->dsrc = src0;
+                if (div0) {
+                        info->ddiv |= 0x80000000;
+                        info->ddiv |= div0 << 8;
+                        info->ddiv |= div0;
+                }
+                if (div1D) {
+                        info->mdiv |= 0x80000000;
+                        info->mdiv |= div1D;
+                }
+                info->ssel = info->coef = 0;
+                info->freq = clk0;
+        } else {
+                if (div1P) {
+                        info->mdiv |= 0x80000000;
+                        info->mdiv |= div1P << 8;
+                }
+                info->ssel = (1 << clk);
+                info->freq = clk1;
+        }
+
+        return 0;
+}
+
+static int
+nvc0_clock_calc(struct nouveau_clock *clk, struct nouveau_cstate *cstate)
+{
+        struct nvc0_clock_priv *priv = (void *)clk;
+        int ret;
+
+        if ((ret = calc_clk(priv, cstate, 0x00, nv_clk_src_gpc)) ||
+            (ret = calc_clk(priv, cstate, 0x01, nv_clk_src_rop)) ||
+            (ret = calc_clk(priv, cstate, 0x02, nv_clk_src_hubk07)) ||
+            (ret = calc_clk(priv, cstate, 0x07, nv_clk_src_hubk06)) ||
+            (ret = calc_clk(priv, cstate, 0x08, nv_clk_src_hubk01)) ||
+            (ret = calc_clk(priv, cstate, 0x09, nv_clk_src_copy)) ||
+            (ret = calc_clk(priv, cstate, 0x0c, nv_clk_src_daemon)) ||
+            (ret = calc_clk(priv, cstate, 0x0e, nv_clk_src_vdec)))
+                return ret;
+
+        return 0;
+}
+
+static void
+nvc0_clock_prog_0(struct nvc0_clock_priv *priv, int clk)
+{
+        struct nvc0_clock_info *info = &priv->eng[clk];
+        if (clk < 7 && !info->ssel) {
+                nv_mask(priv, 0x1371d0 + (clk * 0x04), 0x80003f3f, info->ddiv);
+                nv_wr32(priv, 0x137160 + (clk * 0x04), info->dsrc);
+        }
+}
+
+static void
+nvc0_clock_prog_1(struct nvc0_clock_priv *priv, int clk)
+{
+        nv_mask(priv, 0x137100, (1 << clk), 0x00000000);
+        nv_wait(priv, 0x137100, (1 << clk), 0x00000000);
+}
+
+static void
+nvc0_clock_prog_2(struct nvc0_clock_priv *priv, int clk)
+{
+        struct nvc0_clock_info *info = &priv->eng[clk];
+        const u32 addr = 0x137000 + (clk * 0x20);
+        if (clk <= 7) {
+                nv_mask(priv, addr + 0x00, 0x00000004, 0x00000000);
+                nv_mask(priv, addr + 0x00, 0x00000001, 0x00000000);
+                if (info->coef) {
+                        nv_wr32(priv, addr + 0x04, info->coef);
+                        nv_mask(priv, addr + 0x00, 0x00000001, 0x00000001);
+                        nv_wait(priv, addr + 0x00, 0x00020000, 0x00020000);
+                        nv_mask(priv, addr + 0x00, 0x00020004, 0x00000004);
+                }
+        }
+}
+
+static void
+nvc0_clock_prog_3(struct nvc0_clock_priv *priv, int clk)
+{
+        struct nvc0_clock_info *info = &priv->eng[clk];
+        if (info->ssel) {
+                nv_mask(priv, 0x137100, (1 << clk), info->ssel);
+                nv_wait(priv, 0x137100, (1 << clk), info->ssel);
+        }
+}
+
+static void
+nvc0_clock_prog_4(struct nvc0_clock_priv *priv, int clk)
+{
+        struct nvc0_clock_info *info = &priv->eng[clk];
+        nv_mask(priv, 0x137250 + (clk * 0x04), 0x00003f3f, info->mdiv);
+}
+
+static int
+nvc0_clock_prog(struct nouveau_clock *clk)
+{
+        struct nvc0_clock_priv *priv = (void *)clk;
+        struct {
+                void (*exec)(struct nvc0_clock_priv *, int);
+        } stage[] = {
+                { nvc0_clock_prog_0 }, /* div programming */
+                { nvc0_clock_prog_1 }, /* select div mode */
+                { nvc0_clock_prog_2 }, /* (maybe) program pll */
+                { nvc0_clock_prog_3 }, /* (maybe) select pll mode */
+                { nvc0_clock_prog_4 }, /* final divider */
+        };
+        int i, j;
+
+        for (i = 0; i < ARRAY_SIZE(stage); i++) {
+                for (j = 0; j < ARRAY_SIZE(priv->eng); j++) {
+                        if (!priv->eng[j].freq)
+                                continue;
+                        stage[i].exec(priv, j);
+                }
+        }
+
+        return 0;
+}
+
+static void
+nvc0_clock_tidy(struct nouveau_clock *clk)
+{
+        struct nvc0_clock_priv *priv = (void *)clk;
+        memset(priv->eng, 0x00, sizeof(priv->eng));
+}
+
+static struct nouveau_clocks
+nvc0_domain[] = {
+        { nv_clk_src_crystal, 0xff },
+        { nv_clk_src_href   , 0xff },
+        { nv_clk_src_hubk06 , 0x00 },
+        { nv_clk_src_hubk01 , 0x01 },
+        { nv_clk_src_copy   , 0x02 },
+        { nv_clk_src_gpc    , 0x03, 0, "core", 2000 },
+        { nv_clk_src_rop    , 0x04 },
+        { nv_clk_src_mem    , 0x05, 0, "memory", 1000 },
+        { nv_clk_src_vdec   , 0x06 },
+        { nv_clk_src_daemon , 0x0a },
+        { nv_clk_src_hubk07 , 0x0b },
+        { nv_clk_src_max }
 };
 
 static int
@@ -40,12 +437,15 @@ nvc0_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nvc0_clock_priv *priv;
         int ret;
 
-        ret = nouveau_clock_create(parent, engine, oclass, &priv);
+        ret = nouveau_clock_create(parent, engine, oclass, nvc0_domain, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
 
-        priv->base.pll_calc = nva3_clock_pll_calc;
+        priv->base.read = nvc0_clock_read;
+        priv->base.calc = nvc0_clock_calc;
+        priv->base.prog = nvc0_clock_prog;
+        priv->base.tidy = nvc0_clock_tidy;
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/nve0.c b/drivers/gpu/drm/nouveau/core/subdev/clock/nve0.c
new file mode 100644
index 000000000000..4c62e84b96f5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/nve0.c
@@ -0,0 +1,497 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/clock.h>
+#include <subdev/timer.h>
+#include <subdev/bios.h>
+#include <subdev/bios/pll.h>
+
+#include "pll.h"
+
+struct nve0_clock_info {
+        u32 freq;
+        u32 ssel;
+        u32 mdiv;
+        u32 dsrc;
+        u32 ddiv;
+        u32 coef;
+};
+
+struct nve0_clock_priv {
+        struct nouveau_clock base;
+        struct nve0_clock_info eng[16];
+};
+
+static u32 read_div(struct nve0_clock_priv *, int, u32, u32);
+static u32 read_pll(struct nve0_clock_priv *, u32);
+
+static u32
+read_vco(struct nve0_clock_priv *priv, u32 dsrc)
+{
+        u32 ssrc = nv_rd32(priv, dsrc);
+        if (!(ssrc & 0x00000100))
+                return read_pll(priv, 0x00e800);
+        return read_pll(priv, 0x00e820);
+}
+
+static u32
+read_pll(struct nve0_clock_priv *priv, u32 pll)
+{
+        u32 ctrl = nv_rd32(priv, pll + 0x00);
+        u32 coef = nv_rd32(priv, pll + 0x04);
+        u32 P = (coef & 0x003f0000) >> 16;
+        u32 N = (coef & 0x0000ff00) >> 8;
+        u32 M = (coef & 0x000000ff) >> 0;
+        u32 sclk;
+        u16 fN = 0xf000;
+
+        if (!(ctrl & 0x00000001))
+                return 0;
+
+        switch (pll) {
+        case 0x00e800:
+        case 0x00e820:
+                sclk = nv_device(priv)->crystal;
+                P = 1;
+                break;
+        case 0x132000:
+                sclk = read_pll(priv, 0x132020);
+                P = (coef & 0x10000000) ? 2 : 1;
+                break;
+        case 0x132020:
+                sclk = read_div(priv, 0, 0x137320, 0x137330);
+                fN   = nv_rd32(priv, pll + 0x10) >> 16;
+                break;
+        case 0x137000:
+        case 0x137020:
+        case 0x137040:
+        case 0x1370e0:
+                sclk = read_div(priv, (pll & 0xff) / 0x20, 0x137120, 0x137140);
+                break;
+        default:
+                return 0;
+        }
+
+        if (P == 0)
+                P = 1;
+
+        sclk = (sclk * N) + (((u16)(fN + 4096) * sclk) >> 13);
+        return sclk / (M * P);
+}
+
+static u32
+read_div(struct nve0_clock_priv *priv, int doff, u32 dsrc, u32 dctl)
+{
+        u32 ssrc = nv_rd32(priv, dsrc + (doff * 4));
+        u32 sctl = nv_rd32(priv, dctl + (doff * 4));
+
+        switch (ssrc & 0x00000003) {
+        case 0:
+                if ((ssrc & 0x00030000) != 0x00030000)
+                        return nv_device(priv)->crystal;
+                return 108000;
+        case 2:
+                return 100000;
+        case 3:
+                if (sctl & 0x80000000) {
+                        u32 sclk = read_vco(priv, dsrc + (doff * 4));
+                        u32 sdiv = (sctl & 0x0000003f) + 2;
+                        return (sclk * 2) / sdiv;
+                }
+
+                return read_vco(priv, dsrc + (doff * 4));
+        default:
+                return 0;
+        }
+}
+
+static u32
+read_mem(struct nve0_clock_priv *priv)
+{
+        switch (nv_rd32(priv, 0x1373f4) & 0x0000000f) {
+        case 1: return read_pll(priv, 0x132020);
+        case 2: return read_pll(priv, 0x132000);
+        default:
+                return 0;
+        }
+}
+
+static u32
+read_clk(struct nve0_clock_priv *priv, int clk)
+{
+        u32 sctl = nv_rd32(priv, 0x137250 + (clk * 4));
+        u32 sclk, sdiv;
+
+        if (clk < 7) {
+                u32 ssel = nv_rd32(priv, 0x137100);
+                if (ssel & (1 << clk)) {
+                        sclk = read_pll(priv, 0x137000 + (clk * 0x20));
+                        sdiv = 1;
+                } else {
+                        sclk = read_div(priv, clk, 0x137160, 0x1371d0);
+                        sdiv = 0;
+                }
+        } else {
+                u32 ssrc = nv_rd32(priv, 0x137160 + (clk * 0x04));
+                if ((ssrc & 0x00000003) == 0x00000003) {
+                        sclk = read_div(priv, clk, 0x137160, 0x1371d0);
+                        if (ssrc & 0x00000100) {
+                                if (ssrc & 0x40000000)
+                                        sclk = read_pll(priv, 0x1370e0);
+                                sdiv = 1;
+                        } else {
+                                sdiv = 0;
+                        }
+                } else {
+                        sclk = read_div(priv, clk, 0x137160, 0x1371d0);
+                        sdiv = 0;
+                }
+        }
+
+        if (sctl & 0x80000000) {
+                if (sdiv)
+                        sdiv = ((sctl & 0x00003f00) >> 8) + 2;
+                else
+                        sdiv = ((sctl & 0x0000003f) >> 0) + 2;
+                return (sclk * 2) / sdiv;
+        }
+
+        return sclk;
+}
+
+static int
+nve0_clock_read(struct nouveau_clock *clk, enum nv_clk_src src)
+{
+        struct nouveau_device *device = nv_device(clk);
+        struct nve0_clock_priv *priv = (void *)clk;
+
+        switch (src) {
+        case nv_clk_src_crystal:
+                return device->crystal;
+        case nv_clk_src_href:
+                return 100000;
+        case nv_clk_src_mem:
+                return read_mem(priv);
+        case nv_clk_src_gpc:
+                return read_clk(priv, 0x00);
+        case nv_clk_src_rop:
+                return read_clk(priv, 0x01);
+        case nv_clk_src_hubk07:
+                return read_clk(priv, 0x02);
+        case nv_clk_src_hubk06:
+                return read_clk(priv, 0x07);
+        case nv_clk_src_hubk01:
+                return read_clk(priv, 0x08);
+        case nv_clk_src_daemon:
+                return read_clk(priv, 0x0c);
+        case nv_clk_src_vdec:
+                return read_clk(priv, 0x0e);
+        default:
+                nv_error(clk, "invalid clock source %d\n", src);
+                return -EINVAL;
+        }
+}
+
+static u32
+calc_div(struct nve0_clock_priv *priv, int clk, u32 ref, u32 freq, u32 *ddiv)
+{
+        u32 div = min((ref * 2) / freq, (u32)65);
+        if (div < 2)
+                div = 2;
+
+        *ddiv = div - 2;
+        return (ref * 2) / div;
+}
+
+static u32
+calc_src(struct nve0_clock_priv *priv, int clk, u32 freq, u32 *dsrc, u32 *ddiv)
+{
+        u32 sclk;
+
+        /* use one of the fixed frequencies if possible */
+        *ddiv = 0x00000000;
+        switch (freq) {
+        case  27000:
+        case 108000:
+                *dsrc = 0x00000000;
+                if (freq == 108000)
+                        *dsrc |= 0x00030000;
+                return freq;
+        case 100000:
+                *dsrc = 0x00000002;
+                return freq;
+        default:
+                *dsrc = 0x00000003;
+                break;
+        }
+
+        /* otherwise, calculate the closest divider */
+        sclk = read_vco(priv, 0x137160 + (clk * 4));
+        if (clk < 7)
+                sclk = calc_div(priv, clk, sclk, freq, ddiv);
+        return sclk;
+}
+
+static u32
+calc_pll(struct nve0_clock_priv *priv, int clk, u32 freq, u32 *coef)
+{
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        struct nvbios_pll limits;
+        int N, M, P, ret;
+
+        ret = nvbios_pll_parse(bios, 0x137000 + (clk * 0x20), &limits);
+        if (ret)
+                return 0;
+
+        limits.refclk = read_div(priv, clk, 0x137120, 0x137140);
+        if (!limits.refclk)
+                return 0;
+
+        ret = nva3_pll_calc(nv_subdev(priv), &limits, freq, &N, NULL, &M, &P);
+        if (ret <= 0)
+                return 0;
+
+        *coef = (P << 16) | (N << 8) | M;
+        return ret;
+}
+
+static int
+calc_clk(struct nve0_clock_priv *priv,
+         struct nouveau_cstate *cstate, int clk, int dom)
+{
+        struct nve0_clock_info *info = &priv->eng[clk];
+        u32 freq = cstate->domain[dom];
+        u32 src0, div0, div1D, div1P = 0;
+        u32 clk0, clk1 = 0;
+
+        /* invalid clock domain */
+        if (!freq)
+                return 0;
+
+        /* first possible path, using only dividers */
+        clk0 = calc_src(priv, clk, freq, &src0, &div0);
+        clk0 = calc_div(priv, clk, clk0, freq, &div1D);
+
+        /* see if we can get any closer using PLLs */
+        if (clk0 != freq && (0x0000ff87 & (1 << clk))) {
+                if (clk <= 7)
+                        clk1 = calc_pll(priv, clk, freq, &info->coef);
+                else
+                        clk1 = cstate->domain[nv_clk_src_hubk06];
+                clk1 = calc_div(priv, clk, clk1, freq, &div1P);
+        }
+
+        /* select the method which gets closest to target freq */
+        if (abs((int)freq - clk0) <= abs((int)freq - clk1)) {
+                info->dsrc = src0;
+                if (div0) {
+                        info->ddiv |= 0x80000000;
+                        info->ddiv |= div0 << 8;
+                        info->ddiv |= div0;
+                }
+                if (div1D) {
+                        info->mdiv |= 0x80000000;
+                        info->mdiv |= div1D;
+                }
+                info->ssel = 0;
+                info->freq = clk0;
+        } else {
+                if (div1P) {
+                        info->mdiv |= 0x80000000;
+                        info->mdiv |= div1P << 8;
+                }
+                info->ssel = (1 << clk);
+                info->dsrc = 0x40000100;
+                info->freq = clk1;
+        }
+
+        return 0;
+}
+
+static int
+nve0_clock_calc(struct nouveau_clock *clk, struct nouveau_cstate *cstate)
+{
+        struct nve0_clock_priv *priv = (void *)clk;
+        int ret;
+
+        if ((ret = calc_clk(priv, cstate, 0x00, nv_clk_src_gpc)) ||
+            (ret = calc_clk(priv, cstate, 0x01, nv_clk_src_rop)) ||
+            (ret = calc_clk(priv, cstate, 0x02, nv_clk_src_hubk07)) ||
+            (ret = calc_clk(priv, cstate, 0x07, nv_clk_src_hubk06)) ||
+            (ret = calc_clk(priv, cstate, 0x08, nv_clk_src_hubk01)) ||
+            (ret = calc_clk(priv, cstate, 0x0c, nv_clk_src_daemon)) ||
+            (ret = calc_clk(priv, cstate, 0x0e, nv_clk_src_vdec)))
+                return ret;
+
+        return 0;
+}
+
+static void
+nve0_clock_prog_0(struct nve0_clock_priv *priv, int clk)
+{
+        struct nve0_clock_info *info = &priv->eng[clk];
+        if (!info->ssel) {
+                nv_mask(priv, 0x1371d0 + (clk * 0x04), 0x80003f3f, info->ddiv);
+                nv_wr32(priv, 0x137160 + (clk * 0x04), info->dsrc);
+        }
+}
+
+static void
+nve0_clock_prog_1_0(struct nve0_clock_priv *priv, int clk)
+{
+        nv_mask(priv, 0x137100, (1 << clk), 0x00000000);
+        nv_wait(priv, 0x137100, (1 << clk), 0x00000000);
+}
+
+static void
+nve0_clock_prog_1_1(struct nve0_clock_priv *priv, int clk)
+{
+        nv_mask(priv, 0x137160 + (clk * 0x04), 0x00000100, 0x00000000);
+}
+
+static void
+nve0_clock_prog_2(struct nve0_clock_priv *priv, int clk)
+{
+        struct nve0_clock_info *info = &priv->eng[clk];
+        const u32 addr = 0x137000 + (clk * 0x20);
+        nv_mask(priv, addr + 0x00, 0x00000004, 0x00000000);
+        nv_mask(priv, addr + 0x00, 0x00000001, 0x00000000);
+        if (info->coef) {
+                nv_wr32(priv, addr + 0x04, info->coef);
+                nv_mask(priv, addr + 0x00, 0x00000001, 0x00000001);
+                nv_wait(priv, addr + 0x00, 0x00020000, 0x00020000);
+                nv_mask(priv, addr + 0x00, 0x00020004, 0x00000004);
+        }
+}
+
+static void
+nve0_clock_prog_3(struct nve0_clock_priv *priv, int clk)
+{
+        struct nve0_clock_info *info = &priv->eng[clk];
+        nv_mask(priv, 0x137250 + (clk * 0x04), 0x00003f3f, info->mdiv);
+}
+
+static void
+nve0_clock_prog_4_0(struct nve0_clock_priv *priv, int clk)
+{
+        struct nve0_clock_info *info = &priv->eng[clk];
+        if (info->ssel) {
+                nv_mask(priv, 0x137100, (1 << clk), info->ssel);
+                nv_wait(priv, 0x137100, (1 << clk), info->ssel);
+        }
+}
+
+static void
+nve0_clock_prog_4_1(struct nve0_clock_priv *priv, int clk)
+{
+        struct nve0_clock_info *info = &priv->eng[clk];
+        if (info->ssel) {
+                nv_mask(priv, 0x137160 + (clk * 0x04), 0x40000000, 0x40000000);
+                nv_mask(priv, 0x137160 + (clk * 0x04), 0x00000100, 0x00000100);
+        }
+}
+
+static int
+nve0_clock_prog(struct nouveau_clock *clk)
+{
+        struct nve0_clock_priv *priv = (void *)clk;
+        struct {
+                u32 mask;
+                void (*exec)(struct nve0_clock_priv *, int);
+        } stage[] = {
+                { 0x007f, nve0_clock_prog_0   }, /* div programming */
+                { 0x007f, nve0_clock_prog_1_0 }, /* select div mode */
+                { 0xff80, nve0_clock_prog_1_1 },
+                { 0x00ff, nve0_clock_prog_2   }, /* (maybe) program pll */
+                { 0xff80, nve0_clock_prog_3   }, /* final divider */
+                { 0x007f, nve0_clock_prog_4_0 }, /* (maybe) select pll mode */
+                { 0xff80, nve0_clock_prog_4_1 },
+        };
+        int i, j;
+
+        for (i = 0; i < ARRAY_SIZE(stage); i++) {
+                for (j = 0; j < ARRAY_SIZE(priv->eng); j++) {
+                        if (!(stage[i].mask & (1 << j)))
+                                continue;
+                        if (!priv->eng[j].freq)
+                                continue;
+                        stage[i].exec(priv, j);
+                }
+        }
+
+        return 0;
+}
+
+static void
+nve0_clock_tidy(struct nouveau_clock *clk)
+{
+        struct nve0_clock_priv *priv = (void *)clk;
+        memset(priv->eng, 0x00, sizeof(priv->eng));
+}
+
+static struct nouveau_clocks
+nve0_domain[] = {
+        { nv_clk_src_crystal, 0xff },
+        { nv_clk_src_href   , 0xff },
+        { nv_clk_src_gpc    , 0x00, NVKM_CLK_DOM_FLAG_CORE, "core", 2000 },
+        { nv_clk_src_hubk07 , 0x01, NVKM_CLK_DOM_FLAG_CORE },
+        { nv_clk_src_rop    , 0x02, NVKM_CLK_DOM_FLAG_CORE },
+        { nv_clk_src_mem    , 0x03, 0, "memory", 1000 },
+        { nv_clk_src_hubk06 , 0x04, NVKM_CLK_DOM_FLAG_CORE },
+        { nv_clk_src_hubk01 , 0x05 },
+        { nv_clk_src_vdec   , 0x06 },
+        { nv_clk_src_daemon , 0x07 },
+        { nv_clk_src_max }
+};
+
+static int
+nve0_clock_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+                struct nouveau_oclass *oclass, void *data, u32 size,
+                struct nouveau_object **pobject)
+{
+        struct nve0_clock_priv *priv;
+        int ret;
+
+        ret = nouveau_clock_create(parent, engine, oclass, nve0_domain, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        priv->base.read = nve0_clock_read;
+        priv->base.calc = nve0_clock_calc;
+        priv->base.prog = nve0_clock_prog;
+        priv->base.tidy = nve0_clock_tidy;
+        return 0;
+}
+
+struct nouveau_oclass
+nve0_clock_oclass = {
+        .handle = NV_SUBDEV(CLOCK, 0xe0),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nve0_clock_ctor,
+                .dtor = _nouveau_clock_dtor,
+                .init = _nouveau_clock_init,
+                .fini = _nouveau_clock_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/pllnv04.c b/drivers/gpu/drm/nouveau/core/subdev/clock/pllnv04.c
index cf1ed0dc9bc9..b47d543ab2e3 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/pllnv04.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/pllnv04.c
@@ -38,7 +38,7 @@ getMNP_single(struct nouveau_subdev *subdev, struct nvbios_pll *info, int clk,
          * "clk" parameter in kHz
          * returns calculated clock
          */
-        int cv = nouveau_bios(subdev)->version.chip;
+        struct nouveau_bios *bios = nouveau_bios(subdev);
         int minvco = info->vco1.min_freq, maxvco = info->vco1.max_freq;
         int minM = info->vco1.min_m, maxM = info->vco1.max_m;
         int minN = info->vco1.min_n, maxN = info->vco1.max_n;
@@ -54,18 +54,21 @@ getMNP_single(struct nouveau_subdev *subdev, struct nvbios_pll *info, int clk,
 
         /* this division verified for nv20, nv18, nv28 (Haiku), and nv34 */
         /* possibly correlated with introduction of 27MHz crystal */
-        if (cv < 0x17 || cv == 0x1a || cv == 0x20) {
-                if (clk > 250000)
-                        maxM = 6;
-                if (clk > 340000)
-                        maxM = 2;
-        } else if (cv < 0x40) {
-                if (clk > 150000)
-                        maxM = 6;
-                if (clk > 200000)
-                        maxM = 4;
-                if (clk > 340000)
-                        maxM = 2;
+        if (bios->version.major < 0x60) {
+                int cv = bios->version.chip;
+                if (cv < 0x17 || cv == 0x1a || cv == 0x20) {
+                        if (clk > 250000)
+                                maxM = 6;
+                        if (clk > 340000)
+                                maxM = 2;
+                } else if (cv < 0x40) {
+                        if (clk > 150000)
+                                maxM = 6;
+                        if (clk > 200000)
+                                maxM = 4;
+                        if (clk > 340000)
+                                maxM = 2;
+                }
         }
 
         P = 1 << maxP;
@@ -227,10 +230,12 @@ nv04_pll_calc(struct nouveau_subdev *subdev, struct nvbios_pll *info, u32 freq,
 {
         int ret;
 
-        if (!info->vco2.max_freq) {
+        if (!info->vco2.max_freq || !N2) {
                 ret = getMNP_single(subdev, info, freq, N1, M1, P);
-                *N2 = 1;
-                *M2 = 1;
+                if (N2) {
+                        *N2 = 1;
+                        *M2 = 1;
+                }
         } else {
                 ret = getMNP_double(subdev, info, freq, N1, M1, N2, M2, P);
         }
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/pllnva3.c b/drivers/gpu/drm/nouveau/core/subdev/clock/pllnva3.c
index 2fe1f712eefa..8eca457c2814 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/clock/pllnva3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/pllnva3.c
@@ -45,6 +45,7 @@ nva3_pll_calc(struct nouveau_subdev *subdev, struct nvbios_pll *info,
         lM = max(lM, (int)info->vco1.min_m);
         hM = (info->refclk + info->vco1.min_inputfreq) / info->vco1.min_inputfreq;
         hM = min(hM, (int)info->vco1.max_m);
+        lM = min(lM, hM);
 
         for (M = lM; M <= hM; M++) {
                 u32 tmp = freq * *P * M;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/clock/seq.h b/drivers/gpu/drm/nouveau/core/subdev/clock/seq.h
new file mode 100644
index 000000000000..fb33f06ebd59
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/clock/seq.h
@@ -0,0 +1,17 @@
+#ifndef __NVKM_CLK_SEQ_H__
+#define __NVKM_CLK_SEQ_H__
+
+#include <subdev/bus.h>
+#include <subdev/bus/hwsq.h>
+
+#define clk_init(s,p)       hwsq_init(&(s)->base, (p))
+#define clk_exec(s,e)       hwsq_exec(&(s)->base, (e))
+#define clk_have(s,r)       ((s)->r_##r.addr != 0x000000)
+#define clk_rd32(s,r)       hwsq_rd32(&(s)->base, &(s)->r_##r)
+#define clk_wr32(s,r,d)     hwsq_wr32(&(s)->base, &(s)->r_##r, (d))
+#define clk_mask(s,r,m,d)   hwsq_mask(&(s)->base, &(s)->r_##r, (m), (d))
+#define clk_setf(s,f,d)     hwsq_setf(&(s)->base, (f), (d))
+#define clk_wait(s,f,d)     hwsq_wait(&(s)->base, (f), (d))
+#define clk_nsec(s,n)       hwsq_nsec(&(s)->base, (n))
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c
index b22357d9b821..27c8235f1a85 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c
@@ -168,7 +168,8 @@ setPLL_single(struct nouveau_devinit *devinit, u32 reg,
                 /* downclock -- write new NM first */
                 nv_wr32(devinit, reg, (oldpll & 0xffff0000) | pv->NM1);
 
-        if (chip_version < 0x17 && chip_version != 0x11)
+        if ((chip_version < 0x17 || chip_version == 0x1a) &&
+            chip_version != 0x11)
                 /* wait a bit on older chips */
                 msleep(64);
         nv_rd32(devinit, reg);
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c
index 463b08fa0968..8d274dba1ef1 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c
@@ -38,12 +38,18 @@ static void
 nv10_devinit_meminit(struct nouveau_devinit *devinit)
 {
         struct nv10_devinit_priv *priv = (void *)devinit;
-        const int mem_width[] = { 0x10, 0x00, 0x20 };
-        const int mem_width_count = nv_device(priv)->chipset >= 0x17 ? 3 : 2;
+        static const int mem_width[] = { 0x10, 0x00, 0x20 };
+        int mem_width_count;
         uint32_t patt = 0xdeadbeef;
         struct io_mapping *fb;
         int i, j, k;
 
+        if (nv_device(priv)->card_type >= NV_11 &&
+            nv_device(priv)->chipset >= 0x17)
+                mem_width_count = 3;
+        else
+                mem_width_count = 2;
+
         /* Map the framebuffer aperture */
         fb = fbmem_init(nv_device(priv)->pdev);
         if (!fb) {
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/base.c b/drivers/gpu/drm/nouveau/core/subdev/fb/base.c
index 821cd75b86a3..f009d8a39d9d 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/base.c
@@ -22,9 +22,10 @@
  * Authors: Ben Skeggs
  */
 
-#include "subdev/fb.h"
-#include "subdev/bios.h"
-#include "subdev/bios/bit.h"
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+
+#include "priv.h"
 
 int
 nouveau_fb_bios_memtype(struct nouveau_bios *bios)
@@ -106,9 +107,9 @@ _nouveau_fb_dtor(struct nouveau_object *object)
 
 int
 nouveau_fb_create_(struct nouveau_object *parent, struct nouveau_object *engine,
-                   struct nouveau_oclass *oclass, struct nouveau_oclass *ramcls,
-                   int length, void **pobject)
+                   struct nouveau_oclass *oclass, int length, void **pobject)
 {
+        struct nouveau_fb_impl *impl = (void *)oclass;
         static const char *name[] = {
                 [NV_MEM_TYPE_UNKNOWN] = "unknown",
                 [NV_MEM_TYPE_STOLEN ] = "stolen system memory",
@@ -132,8 +133,10 @@ nouveau_fb_create_(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
+        pfb->memtype_valid = impl->memtype;
+
         ret = nouveau_object_ctor(nv_object(pfb), nv_object(pfb),
-                                  ramcls, NULL, 0, &ram);
+                                  impl->ram, NULL, 0, &ram);
         if (ret) {
                 nv_fatal(pfb, "error detecting memory configuration!!\n");
                 return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/gddr5.c b/drivers/gpu/drm/nouveau/core/subdev/fb/gddr5.c
new file mode 100644
index 000000000000..34f9605ffee6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/gddr5.c
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include <subdev/bios.h>
+#include "priv.h"
+
+int
+nouveau_gddr5_calc(struct nouveau_ram *ram)
+{
+        struct nouveau_bios *bios = nouveau_bios(ram);
+        int pd, lf, xd, vh, vr, vo;
+        int WL, CL, WR, at, dt, ds;
+        int rq = ram->freq < 1000000; /* XXX */
+
+        switch (!!ram->ramcfg.data * ram->ramcfg.version) {
+        case 0x11:
+                pd =  (nv_ro08(bios, ram->ramcfg.data + 0x01) & 0x80) >> 7;
+                lf =  (nv_ro08(bios, ram->ramcfg.data + 0x01) & 0x40) >> 6;
+                xd = !(nv_ro08(bios, ram->ramcfg.data + 0x01) & 0x20);
+                vh =  (nv_ro08(bios, ram->ramcfg.data + 0x02) & 0x10) >> 4;
+                vr =  (nv_ro08(bios, ram->ramcfg.data + 0x02) & 0x04) >> 2;
+                vo =   nv_ro08(bios, ram->ramcfg.data + 0x06) & 0xff;
+                break;
+        default:
+                return -ENOSYS;
+        }
+
+        switch (!!ram->timing.data * ram->timing.version) {
+        case 0x20:
+                WL = (nv_ro16(bios, ram->timing.data + 0x04) & 0x0f80) >> 7;
+                CL =  nv_ro08(bios, ram->timing.data + 0x04) & 0x1f;
+                WR =  nv_ro08(bios, ram->timing.data + 0x0a) & 0x7f;
+                at = (nv_ro08(bios, ram->timing.data + 0x2e) & 0xc0) >> 6;
+                dt =  nv_ro08(bios, ram->timing.data + 0x2e) & 0x03;
+                ds =  nv_ro08(bios, ram->timing.data + 0x2f) & 0x03;
+                break;
+        default:
+                return -ENOSYS;
+        }
+
+        if (WL < 1 || WL > 7 || CL < 5 || CL > 36 || WR < 4 || WR > 35)
+                return -EINVAL;
+        CL -= 5;
+        WR -= 4;
+
+        ram->mr[0] &= ~0xf7f;
+        ram->mr[0] |= (WR & 0x0f) << 8;
+        ram->mr[0] |= (CL & 0x0f) << 3;
+        ram->mr[0] |= (WL & 0x07) << 0;
+
+        ram->mr[1] &= ~0x0bf;
+        ram->mr[1] |= (xd & 0x01) << 7;
+        ram->mr[1] |= (at & 0x03) << 4;
+        ram->mr[1] |= (dt & 0x03) << 2;
+        ram->mr[1] |= (ds & 0x03) << 0;
+
+        ram->mr[3] &= ~0x020;
+        ram->mr[3] |= (rq & 0x01) << 5;
+
+        if (!vo)
+                vo = (ram->mr[6] & 0xff0) >> 4;
+        if (ram->mr[6] & 0x001)
+                pd = 1; /* binary driver does this.. bug? */
+        ram->mr[6] &= ~0xff1;
+        ram->mr[6] |= (vo & 0xff) << 4;
+        ram->mr[6] |= (pd & 0x01) << 0;
+
+        if (!(ram->mr[7] & 0x100))
+                vr = 0; /* binary driver does this.. bug? */
+        ram->mr[7] &= ~0x188;
+        ram->mr[7] |= (vr & 0x01) << 8;
+        ram->mr[7] |= (vh & 0x01) << 7;
+        ram->mr[7] |= (lf & 0x01) << 3;
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv04.c
index 1f103c7b89fa..8309fe33fe84 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv04.c
@@ -22,14 +22,10 @@
  * Authors: Ben Skeggs
  */
 
-#include "priv.h"
+#include "nv04.h"
 
 #define NV04_PFB_CFG0                                           0x00100200
 
-struct nv04_fb_priv {
-        struct nouveau_fb base;
-};
-
 bool
 nv04_fb_memtype_valid(struct nouveau_fb *pfb, u32 tile_flags)
 {
@@ -57,30 +53,37 @@ nv04_fb_init(struct nouveau_object *object)
         return 0;
 }
 
-static int
+int
 nv04_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
              struct nouveau_oclass *oclass, void *data, u32 size,
              struct nouveau_object **pobject)
 {
+        struct nv04_fb_impl *impl = (void *)oclass;
         struct nv04_fb_priv *priv;
         int ret;
 
-        ret = nouveau_fb_create(parent, engine, oclass, &nv04_ram_oclass, &priv);
+        ret = nouveau_fb_create(parent, engine, oclass, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
 
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
+        priv->base.tile.regions = impl->tile.regions;
+        priv->base.tile.init = impl->tile.init;
+        priv->base.tile.comp = impl->tile.comp;
+        priv->base.tile.fini = impl->tile.fini;
+        priv->base.tile.prog = impl->tile.prog;
         return 0;
 }
 
-struct nouveau_oclass
-nv04_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x04),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nv04_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x04),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = nv04_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv04_ram_oclass,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv04.h b/drivers/gpu/drm/nouveau/core/subdev/fb/nv04.h
new file mode 100644
index 000000000000..06ce71f87a74
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv04.h
@@ -0,0 +1,55 @@
+#ifndef __NVKM_FB_NV04_H__
+#define __NVKM_FB_NV04_H__
+
+#include "priv.h"
+
+struct nv04_fb_priv {
+        struct nouveau_fb base;
+};
+
+int  nv04_fb_ctor(struct nouveau_object *, struct nouveau_object *,
+                  struct nouveau_oclass *, void *, u32,
+                  struct nouveau_object **);
+
+struct nv04_fb_impl {
+        struct nouveau_fb_impl base;
+        struct {
+                int regions;
+                void (*init)(struct nouveau_fb *, int i, u32 addr, u32 size,
+                             u32 pitch, u32 flags, struct nouveau_fb_tile *);
+                void (*comp)(struct nouveau_fb *, int i, u32 size, u32 flags,
+                             struct nouveau_fb_tile *);
+                void (*fini)(struct nouveau_fb *, int i,
+                             struct nouveau_fb_tile *);
+                void (*prog)(struct nouveau_fb *, int i,
+                             struct nouveau_fb_tile *);
+        } tile;
+};
+
+void nv10_fb_tile_init(struct nouveau_fb *, int i, u32 addr, u32 size,
+                       u32 pitch, u32 flags, struct nouveau_fb_tile *);
+void nv10_fb_tile_fini(struct nouveau_fb *, int i, struct nouveau_fb_tile *);
+void nv10_fb_tile_prog(struct nouveau_fb *, int, struct nouveau_fb_tile *);
+
+void nv20_fb_tile_init(struct nouveau_fb *, int i, u32 addr, u32 size,
+                       u32 pitch, u32 flags, struct nouveau_fb_tile *);
+void nv20_fb_tile_fini(struct nouveau_fb *, int i, struct nouveau_fb_tile *);
+void nv20_fb_tile_prog(struct nouveau_fb *, int, struct nouveau_fb_tile *);
+
+int  nv30_fb_init(struct nouveau_object *);
+void nv30_fb_tile_init(struct nouveau_fb *, int i, u32 addr, u32 size,
+                       u32 pitch, u32 flags, struct nouveau_fb_tile *);
+
+void nv40_fb_tile_comp(struct nouveau_fb *, int i, u32 size, u32 flags,
+                       struct nouveau_fb_tile *);
+
+int  nv41_fb_init(struct nouveau_object *);
+void nv41_fb_tile_prog(struct nouveau_fb *, int, struct nouveau_fb_tile *);
+
+int  nv44_fb_init(struct nouveau_object *);
+void nv44_fb_tile_prog(struct nouveau_fb *, int, struct nouveau_fb_tile *);
+
+void nv46_fb_tile_init(struct nouveau_fb *, int i, u32 addr, u32 size,
+                       u32 pitch, u32 flags, struct nouveau_fb_tile *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv10.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv10.c
index be069b5306b6..ffb7ec6d97aa 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv10.c
@@ -24,11 +24,7 @@
  *
  */
 
-#include "priv.h"
-
-struct nv10_fb_priv {
-        struct nouveau_fb base;
-};
+#include "nv04.h"
 
 void
 nv10_fb_tile_init(struct nouveau_fb *pfb, int i, u32 addr, u32 size, u32 pitch,
@@ -57,34 +53,19 @@ nv10_fb_tile_prog(struct nouveau_fb *pfb, int i, struct nouveau_fb_tile *tile)
         nv_rd32(pfb, 0x100240 + (i * 0x10));
 }
 
-static int
-nv10_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv10_fb_priv *priv;
-        int ret;
-
-        ret = nouveau_fb_create(parent, engine, oclass, &nv10_ram_oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
-        priv->base.tile.regions = 8;
-        priv->base.tile.init = nv10_fb_tile_init;
-        priv->base.tile.fini = nv10_fb_tile_fini;
-        priv->base.tile.prog = nv10_fb_tile_prog;
-        return 0;
-}
-
-struct nouveau_oclass
-nv10_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x10),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv10_fb_ctor,
+struct nouveau_oclass *
+nv10_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x10),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = _nouveau_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv10_ram_oclass,
+        .tile.regions = 8,
+        .tile.init = nv10_fb_tile_init,
+        .tile.fini = nv10_fb_tile_fini,
+        .tile.prog = nv10_fb_tile_prog,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv1a.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv1a.c
index 57a2af0079b3..9159a5ccee93 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv1a.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv1a.c
@@ -24,40 +24,21 @@
  *
  */
 
-#include "priv.h"
+#include "nv04.h"
 
-struct nv1a_fb_priv {
-        struct nouveau_fb base;
-};
-
-static int
-nv1a_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv1a_fb_priv *priv;
-        int ret;
-
-        ret = nouveau_fb_create(parent, engine, oclass, &nv1a_ram_oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
-        priv->base.tile.regions = 8;
-        priv->base.tile.init = nv10_fb_tile_init;
-        priv->base.tile.fini = nv10_fb_tile_fini;
-        priv->base.tile.prog = nv10_fb_tile_prog;
-        return 0;
-}
-
-struct nouveau_oclass
-nv1a_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x1a),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv1a_fb_ctor,
+struct nouveau_oclass *
+nv1a_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x1a),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = _nouveau_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv10_ram_oclass,
+        .tile.regions = 8,
+        .tile.init = nv10_fb_tile_init,
+        .tile.fini = nv10_fb_tile_fini,
+        .tile.prog = nv10_fb_tile_prog,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv20.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv20.c
index b18c4e63bb47..f003c1b1893f 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv20.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv20.c
@@ -24,11 +24,7 @@
  *
  */
 
-#include "priv.h"
-
-struct nv20_fb_priv {
-        struct nouveau_fb base;
-};
+#include "nv04.h"
 
 void
 nv20_fb_tile_init(struct nouveau_fb *pfb, int i, u32 addr, u32 size, u32 pitch,
@@ -80,35 +76,20 @@ nv20_fb_tile_prog(struct nouveau_fb *pfb, int i, struct nouveau_fb_tile *tile)
         nv_wr32(pfb, 0x100300 + (i * 0x04), tile->zcomp);
 }
 
-static int
-nv20_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv20_fb_priv *priv;
-        int ret;
-
-        ret = nouveau_fb_create(parent, engine, oclass, &nv20_ram_oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
-        priv->base.tile.regions = 8;
-        priv->base.tile.init = nv20_fb_tile_init;
-        priv->base.tile.comp = nv20_fb_tile_comp;
-        priv->base.tile.fini = nv20_fb_tile_fini;
-        priv->base.tile.prog = nv20_fb_tile_prog;
-        return 0;
-}
-
-struct nouveau_oclass
-nv20_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x20),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv20_fb_ctor,
+struct nouveau_oclass *
+nv20_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x20),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = _nouveau_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv20_ram_oclass,
+        .tile.regions = 8,
+        .tile.init = nv20_fb_tile_init,
+        .tile.comp = nv20_fb_tile_comp,
+        .tile.fini = nv20_fb_tile_fini,
+        .tile.prog = nv20_fb_tile_prog,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv25.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv25.c
index 32ccabf10c45..f34f4223210b 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv25.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv25.c
@@ -24,11 +24,7 @@
  *
  */
 
-#include "priv.h"
-
-struct nv25_fb_priv {
-        struct nouveau_fb base;
-};
+#include "nv04.h"
 
 static void
 nv25_fb_tile_comp(struct nouveau_fb *pfb, int i, u32 size, u32 flags,
@@ -46,35 +42,20 @@ nv25_fb_tile_comp(struct nouveau_fb *pfb, int i, u32 size, u32 flags,
         }
 }
 
-static int
-nv25_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv25_fb_priv *priv;
-        int ret;
-
-        ret = nouveau_fb_create(parent, engine, oclass, &nv20_ram_oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
-        priv->base.tile.regions = 8;
-        priv->base.tile.init = nv20_fb_tile_init;
-        priv->base.tile.comp = nv25_fb_tile_comp;
-        priv->base.tile.fini = nv20_fb_tile_fini;
-        priv->base.tile.prog = nv20_fb_tile_prog;
-        return 0;
-}
-
-struct nouveau_oclass
-nv25_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x25),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv25_fb_ctor,
+struct nouveau_oclass *
+nv25_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x25),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = _nouveau_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv20_ram_oclass,
+        .tile.regions = 8,
+        .tile.init = nv20_fb_tile_init,
+        .tile.comp = nv25_fb_tile_comp,
+        .tile.fini = nv20_fb_tile_fini,
+        .tile.prog = nv20_fb_tile_prog,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv30.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv30.c
index bef756d43d33..69093f7151f0 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv30.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv30.c
@@ -24,11 +24,7 @@
  *
  */
 
-#include "priv.h"
-
-struct nv30_fb_priv {
-        struct nouveau_fb base;
-};
+#include "nv04.h"
 
 void
 nv30_fb_tile_init(struct nouveau_fb *pfb, int i, u32 addr, u32 size, u32 pitch,
@@ -67,7 +63,7 @@ nv30_fb_tile_comp(struct nouveau_fb *pfb, int i, u32 size, u32 flags,
 }
 
 static int
-calc_bias(struct nv30_fb_priv *priv, int k, int i, int j)
+calc_bias(struct nv04_fb_priv *priv, int k, int i, int j)
 {
         struct nouveau_device *device = nv_device(priv);
         int b = (device->chipset > 0x30 ?
@@ -78,7 +74,7 @@ calc_bias(struct nv30_fb_priv *priv, int k, int i, int j)
 }
 
 static int
-calc_ref(struct nv30_fb_priv *priv, int l, int k, int i)
+calc_ref(struct nv04_fb_priv *priv, int l, int k, int i)
 {
         int j, x = 0;
 
@@ -95,7 +91,7 @@ int
 nv30_fb_init(struct nouveau_object *object)
 {
         struct nouveau_device *device = nv_device(object);
-        struct nv30_fb_priv *priv = (void *)object;
+        struct nv04_fb_priv *priv = (void *)object;
         int ret, i, j;
 
         ret = nouveau_fb_init(&priv->base);
@@ -124,35 +120,20 @@ nv30_fb_init(struct nouveau_object *object)
         return 0;
 }
 
-static int
-nv30_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv30_fb_priv *priv;
-        int ret;
-
-        ret = nouveau_fb_create(parent, engine, oclass, &nv20_ram_oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
-        priv->base.tile.regions = 8;
-        priv->base.tile.init = nv30_fb_tile_init;
-        priv->base.tile.comp = nv30_fb_tile_comp;
-        priv->base.tile.fini = nv20_fb_tile_fini;
-        priv->base.tile.prog = nv20_fb_tile_prog;
-        return 0;
-}
-
-struct nouveau_oclass
-nv30_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x30),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv30_fb_ctor,
+struct nouveau_oclass *
+nv30_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x30),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = nv30_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv20_ram_oclass,
+        .tile.regions = 8,
+        .tile.init = nv30_fb_tile_init,
+        .tile.comp = nv30_fb_tile_comp,
+        .tile.fini = nv20_fb_tile_fini,
+        .tile.prog = nv20_fb_tile_prog,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv35.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv35.c
index 097d8e3824f2..161b06e8fc3f 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv35.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv35.c
@@ -24,11 +24,7 @@
  *
  */
 
-#include "priv.h"
-
-struct nv35_fb_priv {
-        struct nouveau_fb base;
-};
+#include "nv04.h"
 
 static void
 nv35_fb_tile_comp(struct nouveau_fb *pfb, int i, u32 size, u32 flags,
@@ -47,35 +43,20 @@ nv35_fb_tile_comp(struct nouveau_fb *pfb, int i, u32 size, u32 flags,
         }
 }
 
-static int
-nv35_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv35_fb_priv *priv;
-        int ret;
-
-        ret = nouveau_fb_create(parent, engine, oclass, &nv20_ram_oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
-        priv->base.tile.regions = 8;
-        priv->base.tile.init = nv30_fb_tile_init;
-        priv->base.tile.comp = nv35_fb_tile_comp;
-        priv->base.tile.fini = nv20_fb_tile_fini;
-        priv->base.tile.prog = nv20_fb_tile_prog;
-        return 0;
-}
-
-struct nouveau_oclass
-nv35_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x35),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv35_fb_ctor,
+struct nouveau_oclass *
+nv35_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x35),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = nv30_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv20_ram_oclass,
+        .tile.regions = 8,
+        .tile.init = nv30_fb_tile_init,
+        .tile.comp = nv35_fb_tile_comp,
+        .tile.fini = nv20_fb_tile_fini,
+        .tile.prog = nv20_fb_tile_prog,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv36.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv36.c
index 9d6d9df896d9..2dd3d0aab6bb 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv36.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv36.c
@@ -24,11 +24,7 @@
  *
  */
 
-#include "priv.h"
-
-struct nv36_fb_priv {
-        struct nouveau_fb base;
-};
+#include "nv04.h"
 
 static void
 nv36_fb_tile_comp(struct nouveau_fb *pfb, int i, u32 size, u32 flags,
@@ -47,35 +43,20 @@ nv36_fb_tile_comp(struct nouveau_fb *pfb, int i, u32 size, u32 flags,
         }
 }
 
-static int
-nv36_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv36_fb_priv *priv;
-        int ret;
-
-        ret = nouveau_fb_create(parent, engine, oclass, &nv20_ram_oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
-        priv->base.tile.regions = 8;
-        priv->base.tile.init = nv30_fb_tile_init;
-        priv->base.tile.comp = nv36_fb_tile_comp;
-        priv->base.tile.fini = nv20_fb_tile_fini;
-        priv->base.tile.prog = nv20_fb_tile_prog;
-        return 0;
-}
-
-struct nouveau_oclass
-nv36_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x36),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv36_fb_ctor,
+struct nouveau_oclass *
+nv36_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x36),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = nv30_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv20_ram_oclass,
+        .tile.regions = 8,
+        .tile.init = nv30_fb_tile_init,
+        .tile.comp = nv36_fb_tile_comp,
+        .tile.fini = nv20_fb_tile_fini,
+        .tile.prog = nv20_fb_tile_prog,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv40.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv40.c
index 33b4393a7829..95a115ab0c86 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv40.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv40.c
@@ -24,11 +24,7 @@
  *
  */
 
-#include "priv.h"
-
-struct nv40_fb_priv {
-        struct nouveau_fb base;
-};
+#include "nv04.h"
 
 void
 nv40_fb_tile_comp(struct nouveau_fb *pfb, int i, u32 size, u32 flags,
@@ -50,7 +46,7 @@ nv40_fb_tile_comp(struct nouveau_fb *pfb, int i, u32 size, u32 flags,
 static int
 nv40_fb_init(struct nouveau_object *object)
 {
-        struct nv40_fb_priv *priv = (void *)object;
+        struct nv04_fb_priv *priv = (void *)object;
         int ret;
 
         ret = nouveau_fb_init(&priv->base);
@@ -61,36 +57,20 @@ nv40_fb_init(struct nouveau_object *object)
         return 0;
 }
 
-static int
-nv40_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv40_fb_priv *priv;
-        int ret;
-
-        ret = nouveau_fb_create(parent, engine, oclass, &nv40_ram_oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
-        priv->base.tile.regions = 8;
-        priv->base.tile.init = nv30_fb_tile_init;
-        priv->base.tile.comp = nv40_fb_tile_comp;
-        priv->base.tile.fini = nv20_fb_tile_fini;
-        priv->base.tile.prog = nv20_fb_tile_prog;
-        return 0;
-}
-
-
-struct nouveau_oclass
-nv40_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x40),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv40_fb_ctor,
+struct nouveau_oclass *
+nv40_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x40),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = nv40_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv40_ram_oclass,
+        .tile.regions = 8,
+        .tile.init = nv30_fb_tile_init,
+        .tile.comp = nv40_fb_tile_comp,
+        .tile.fini = nv20_fb_tile_fini,
+        .tile.prog = nv20_fb_tile_prog,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv40.h b/drivers/gpu/drm/nouveau/core/subdev/fb/nv40.h
new file mode 100644
index 000000000000..581f808527f2
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv40.h
@@ -0,0 +1,17 @@
+#ifndef __NVKM_FB_NV40_H__
+#define __NVKM_FB_NV40_H__
+
+#include "priv.h"
+
+struct nv40_ram {
+        struct nouveau_ram base;
+        u32 ctrl;
+        u32 coef;
+};
+
+
+int  nv40_ram_calc(struct nouveau_fb *, u32);
+int  nv40_ram_prog(struct nouveau_fb *);
+void nv40_ram_tidy(struct nouveau_fb *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv41.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv41.c
index 02cd83789cd4..b239a8615599 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv41.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv41.c
@@ -24,11 +24,7 @@
  *
  */
 
-#include "priv.h"
-
-struct nv41_fb_priv {
-        struct nouveau_fb base;
-};
+#include "nv04.h"
 
 void
 nv41_fb_tile_prog(struct nouveau_fb *pfb, int i, struct nouveau_fb_tile *tile)
@@ -43,7 +39,7 @@ nv41_fb_tile_prog(struct nouveau_fb *pfb, int i, struct nouveau_fb_tile *tile)
 int
 nv41_fb_init(struct nouveau_object *object)
 {
-        struct nv41_fb_priv *priv = (void *)object;
+        struct nv04_fb_priv *priv = (void *)object;
         int ret;
 
         ret = nouveau_fb_init(&priv->base);
@@ -54,36 +50,20 @@ nv41_fb_init(struct nouveau_object *object)
         return 0;
 }
 
-static int
-nv41_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv41_fb_priv *priv;
-        int ret;
-
-        ret = nouveau_fb_create(parent, engine, oclass, &nv41_ram_oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
-        priv->base.tile.regions = 12;
-        priv->base.tile.init = nv30_fb_tile_init;
-        priv->base.tile.comp = nv40_fb_tile_comp;
-        priv->base.tile.fini = nv20_fb_tile_fini;
-        priv->base.tile.prog = nv41_fb_tile_prog;
-        return 0;
-}
-
-
-struct nouveau_oclass
-nv41_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x41),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv41_fb_ctor,
+struct nouveau_oclass *
+nv41_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x41),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = nv41_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv41_ram_oclass,
+        .tile.regions = 12,
+        .tile.init = nv30_fb_tile_init,
+        .tile.comp = nv40_fb_tile_comp,
+        .tile.fini = nv20_fb_tile_fini,
+        .tile.prog = nv41_fb_tile_prog,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv44.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv44.c
index c5246c29f293..d8478208a681 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv44.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv44.c
@@ -24,11 +24,7 @@
  *
  */
 
-#include "priv.h"
-
-struct nv44_fb_priv {
-        struct nouveau_fb base;
-};
+#include "nv04.h"
 
 static void
 nv44_fb_tile_init(struct nouveau_fb *pfb, int i, u32 addr, u32 size, u32 pitch,
@@ -52,7 +48,7 @@ nv44_fb_tile_prog(struct nouveau_fb *pfb, int i, struct nouveau_fb_tile *tile)
 int
 nv44_fb_init(struct nouveau_object *object)
 {
-        struct nv44_fb_priv *priv = (void *)object;
+        struct nv04_fb_priv *priv = (void *)object;
         int ret;
 
         ret = nouveau_fb_init(&priv->base);
@@ -64,35 +60,19 @@ nv44_fb_init(struct nouveau_object *object)
         return 0;
 }
 
-static int
-nv44_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv44_fb_priv *priv;
-        int ret;
-
-        ret = nouveau_fb_create(parent, engine, oclass, &nv44_ram_oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
-        priv->base.tile.regions = 12;
-        priv->base.tile.init = nv44_fb_tile_init;
-        priv->base.tile.fini = nv20_fb_tile_fini;
-        priv->base.tile.prog = nv44_fb_tile_prog;
-        return 0;
-}
-
-
-struct nouveau_oclass
-nv44_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x44),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv44_fb_ctor,
+struct nouveau_oclass *
+nv44_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x44),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = nv44_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv44_ram_oclass,
+        .tile.regions = 12,
+        .tile.init = nv44_fb_tile_init,
+        .tile.fini = nv20_fb_tile_fini,
+        .tile.prog = nv44_fb_tile_prog,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv46.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv46.c
index e2b57909bfca..a5b77514d35b 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv46.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv46.c
@@ -24,11 +24,7 @@
  *
  */
 
-#include "priv.h"
-
-struct nv46_fb_priv {
-        struct nouveau_fb base;
-};
+#include "nv04.h"
 
 void
 nv46_fb_tile_init(struct nouveau_fb *pfb, int i, u32 addr, u32 size, u32 pitch,
@@ -44,35 +40,19 @@ nv46_fb_tile_init(struct nouveau_fb *pfb, int i, u32 addr, u32 size, u32 pitch,
         tile->pitch = pitch;
 }
 
-static int
-nv46_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv46_fb_priv *priv;
-        int ret;
-
-        ret = nouveau_fb_create(parent, engine, oclass, &nv44_ram_oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
-        priv->base.tile.regions = 15;
-        priv->base.tile.init = nv46_fb_tile_init;
-        priv->base.tile.fini = nv20_fb_tile_fini;
-        priv->base.tile.prog = nv44_fb_tile_prog;
-        return 0;
-}
-
-
-struct nouveau_oclass
-nv46_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x46),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv46_fb_ctor,
+struct nouveau_oclass *
+nv46_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x46),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = nv44_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv44_ram_oclass,
+        .tile.regions = 15,
+        .tile.init = nv46_fb_tile_init,
+        .tile.fini = nv20_fb_tile_fini,
+        .tile.prog = nv44_fb_tile_prog,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv47.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv47.c
index fe6a2278621d..3bea142376bc 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv47.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv47.c
@@ -24,42 +24,22 @@
  *
  */
 
-#include "priv.h"
+#include "nv04.h"
 
-struct nv47_fb_priv {
-        struct nouveau_fb base;
-};
-
-static int
-nv47_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv47_fb_priv *priv;
-        int ret;
-
-        ret = nouveau_fb_create(parent, engine, oclass, &nv41_ram_oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
-        priv->base.tile.regions = 15;
-        priv->base.tile.init = nv30_fb_tile_init;
-        priv->base.tile.comp = nv40_fb_tile_comp;
-        priv->base.tile.fini = nv20_fb_tile_fini;
-        priv->base.tile.prog = nv41_fb_tile_prog;
-        return 0;
-}
-
-
-struct nouveau_oclass
-nv47_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x47),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv47_fb_ctor,
+struct nouveau_oclass *
+nv47_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x47),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = nv41_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv41_ram_oclass,
+        .tile.regions = 15,
+        .tile.init = nv30_fb_tile_init,
+        .tile.comp = nv40_fb_tile_comp,
+        .tile.fini = nv20_fb_tile_fini,
+        .tile.prog = nv41_fb_tile_prog,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv49.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv49.c
index 5eca99b8c7e2..666cbd5d47f5 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv49.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv49.c
@@ -24,42 +24,22 @@
  *
  */
 
-#include "priv.h"
+#include "nv04.h"
 
-struct nv49_fb_priv {
-        struct nouveau_fb base;
-};
-
-static int
-nv49_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv49_fb_priv *priv;
-        int ret;
-
-        ret = nouveau_fb_create(parent, engine, oclass, &nv49_ram_oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
-        priv->base.tile.regions = 15;
-        priv->base.tile.init = nv30_fb_tile_init;
-        priv->base.tile.comp = nv40_fb_tile_comp;
-        priv->base.tile.fini = nv20_fb_tile_fini;
-        priv->base.tile.prog = nv41_fb_tile_prog;
-        return 0;
-}
-
-
-struct nouveau_oclass
-nv49_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x49),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv49_fb_ctor,
+struct nouveau_oclass *
+nv49_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x49),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = nv41_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv49_ram_oclass,
+        .tile.regions = 15,
+        .tile.init = nv30_fb_tile_init,
+        .tile.comp = nv40_fb_tile_comp,
+        .tile.fini = nv20_fb_tile_fini,
+        .tile.prog = nv41_fb_tile_prog,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv4e.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv4e.c
index 1190b78a1e91..42e64f364ec1 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv4e.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv4e.c
@@ -24,40 +24,21 @@
  *
  */
 
-#include "priv.h"
+#include "nv04.h"
 
-struct nv4e_fb_priv {
-        struct nouveau_fb base;
-};
-
-static int
-nv4e_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv4e_fb_priv *priv;
-        int ret;
-
-        ret = nouveau_fb_create(parent, engine, oclass, &nv4e_ram_oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.memtype_valid = nv04_fb_memtype_valid;
-        priv->base.tile.regions = 12;
-        priv->base.tile.init = nv46_fb_tile_init;
-        priv->base.tile.fini = nv20_fb_tile_fini;
-        priv->base.tile.prog = nv44_fb_tile_prog;
-        return 0;
-}
-
-struct nouveau_oclass
-nv4e_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x4e),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv4e_fb_ctor,
+struct nouveau_oclass *
+nv4e_fb_oclass = &(struct nv04_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x4e),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_fb_ctor,
                 .dtor = _nouveau_fb_dtor,
                 .init = nv44_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv04_fb_memtype_valid,
+        .base.ram = &nv4e_ram_oclass,
+        .tile.regions = 12,
+        .tile.init = nv46_fb_tile_init,
+        .tile.fini = nv20_fb_tile_fini,
+        .tile.prog = nv44_fb_tile_prog,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c
index da614ec5564b..cbc7f00c1278 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c
@@ -27,14 +27,9 @@
 #include <core/engctx.h>
 #include <core/object.h>
 
-#include "priv.h"
 #include <subdev/bios.h>
 
-struct nv50_fb_priv {
-        struct nouveau_fb base;
-        struct page *r100c08_page;
-        dma_addr_t r100c08;
-};
+#include "nv50.h"
 
 int
 nv50_fb_memtype[0x80] = {
@@ -48,7 +43,7 @@ nv50_fb_memtype[0x80] = {
         1, 0, 2, 0, 1, 0, 2, 0, 1, 1, 2, 2, 1, 1, 0, 0
 };
 
-static bool
+bool
 nv50_fb_memtype_valid(struct nouveau_fb *pfb, u32 memtype)
 {
         return nv50_fb_memtype[(memtype & 0xff00) >> 8] != 0;
@@ -239,7 +234,7 @@ nv50_fb_intr(struct nouveau_subdev *subdev)
                 pr_cont("0x%08x\n", st1);
 }
 
-static int
+int
 nv50_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
              struct nouveau_oclass *oclass, void *data, u32 size,
              struct nouveau_object **pobject)
@@ -248,7 +243,7 @@ nv50_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nv50_fb_priv *priv;
         int ret;
 
-        ret = nouveau_fb_create(parent, engine, oclass, &nv50_ram_oclass, &priv);
+        ret = nouveau_fb_create(parent, engine, oclass, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
@@ -264,12 +259,11 @@ nv50_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                 nv_warn(priv, "failed 0x100c08 page alloc\n");
         }
 
-        priv->base.memtype_valid = nv50_fb_memtype_valid;
         nv_subdev(priv)->intr = nv50_fb_intr;
         return 0;
 }
 
-static void
+void
 nv50_fb_dtor(struct nouveau_object *object)
 {
         struct nouveau_device *device = nv_device(object);
@@ -284,10 +278,10 @@ nv50_fb_dtor(struct nouveau_object *object)
         nouveau_fb_destroy(&priv->base);
 }
 
-static int
+int
 nv50_fb_init(struct nouveau_object *object)
 {
-        struct nouveau_device *device = nv_device(object);
+        struct nv50_fb_impl *impl = (void *)object->oclass;
         struct nv50_fb_priv *priv = (void *)object;
         int ret;
 
@@ -303,33 +297,20 @@ nv50_fb_init(struct nouveau_object *object)
 
         /* This is needed to get meaningful information from 100c90
          * on traps. No idea what these values mean exactly. */
-        switch (device->chipset) {
-        case 0x50:
-                nv_wr32(priv, 0x100c90, 0x000707ff);
-                break;
-        case 0xa3:
-        case 0xa5:
-        case 0xa8:
-                nv_wr32(priv, 0x100c90, 0x000d0fff);
-                break;
-        case 0xaf:
-                nv_wr32(priv, 0x100c90, 0x089d1fff);
-                break;
-        default:
-                nv_wr32(priv, 0x100c90, 0x001d07ff);
-                break;
-        }
-
+        nv_wr32(priv, 0x100c90, impl->trap);
         return 0;
 }
 
-struct nouveau_oclass
-nv50_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0x50),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nv50_fb_oclass = &(struct nv50_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x50),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv50_fb_ctor,
                 .dtor = nv50_fb_dtor,
                 .init = nv50_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .base.memtype = nv50_fb_memtype_valid,
+        .base.ram = &nv50_ram_oclass,
+        .trap = 0x000707ff,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.h b/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.h
new file mode 100644
index 000000000000..c5e5a888c607
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.h
@@ -0,0 +1,33 @@
+#ifndef __NVKM_FB_NV50_H__
+#define __NVKM_FB_NV50_H__
+
+#include "priv.h"
+
+struct nv50_fb_priv {
+        struct nouveau_fb base;
+        struct page *r100c08_page;
+        dma_addr_t r100c08;
+};
+
+int  nv50_fb_ctor(struct nouveau_object *, struct nouveau_object *,
+                  struct nouveau_oclass *, void *, u32,
+                  struct nouveau_object **);
+void nv50_fb_dtor(struct nouveau_object *);
+int  nv50_fb_init(struct nouveau_object *);
+
+struct nv50_fb_impl {
+        struct nouveau_fb_impl base;
+        u32 trap;
+};
+
+#define nv50_ram_create(p,e,o,d)                                               \
+        nv50_ram_create_((p), (e), (o), sizeof(**d), (void **)d)
+int  nv50_ram_create_(struct nouveau_object *, struct nouveau_object *,
+                      struct nouveau_oclass *, int, void **);
+int  nv50_ram_get(struct nouveau_fb *, u64 size, u32 align, u32 ncmin,
+                  u32 memtype, struct nouveau_mem **);
+void nv50_ram_put(struct nouveau_fb *, struct nouveau_mem **);
+void __nv50_ram_put(struct nouveau_fb *, struct nouveau_mem *);
+extern int nv50_fb_memtype[0x80];
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv84.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv84.c
new file mode 100644
index 000000000000..cf0e767d3833
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv84.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv50.h"
+
+struct nouveau_oclass *
+nv84_fb_oclass = &(struct nv50_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0x84),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv50_fb_ctor,
+                .dtor = nv50_fb_dtor,
+                .init = nv50_fb_init,
+                .fini = _nouveau_fb_fini,
+        },
+        .base.memtype = nv50_fb_memtype_valid,
+        .base.ram = &nv50_ram_oclass,
+        .trap = 0x001d07ff,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nva3.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nva3.c
new file mode 100644
index 000000000000..dab6e1c63d48
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nva3.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv50.h"
+
+struct nouveau_oclass *
+nva3_fb_oclass = &(struct nv50_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0xa3),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv50_fb_ctor,
+                .dtor = nv50_fb_dtor,
+                .init = nv50_fb_init,
+                .fini = _nouveau_fb_fini,
+        },
+        .base.memtype = nv50_fb_memtype_valid,
+        .base.ram = &nva3_ram_oclass,
+        .trap = 0x000d0fff,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nvaa.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nvaa.c
new file mode 100644
index 000000000000..cba8e6818035
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nvaa.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv50.h"
+
+struct nouveau_oclass *
+nvaa_fb_oclass = &(struct nv50_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0xaa),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv50_fb_ctor,
+                .dtor = nv50_fb_dtor,
+                .init = nv50_fb_init,
+                .fini = _nouveau_fb_fini,
+        },
+        .base.memtype = nv50_fb_memtype_valid,
+        .base.ram = &nvaa_ram_oclass,
+        .trap = 0x001d07ff,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nvaf.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nvaf.c
new file mode 100644
index 000000000000..5423faa2c09b
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nvaf.c
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv50.h"
+
+struct nouveau_oclass *
+nvaf_fb_oclass = &(struct nv50_fb_impl) {
+        .base.base.handle = NV_SUBDEV(FB, 0xaf),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv50_fb_ctor,
+                .dtor = nv50_fb_dtor,
+                .init = nv50_fb_init,
+                .fini = _nouveau_fb_fini,
+        },
+        .base.memtype = nv50_fb_memtype_valid,
+        .base.ram = &nvaa_ram_oclass,
+        .trap = 0x089d1fff,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c
index f35d76fd746d..e5fc37c4caac 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c
@@ -22,24 +22,18 @@
  * Authors: Ben Skeggs
  */
 
-#include "priv.h"
-
-struct nvc0_fb_priv {
-        struct nouveau_fb base;
-        struct page *r100c10_page;
-        dma_addr_t r100c10;
-};
+#include "nvc0.h"
 
 extern const u8 nvc0_pte_storage_type_map[256];
 
-static bool
+bool
 nvc0_fb_memtype_valid(struct nouveau_fb *pfb, u32 tile_flags)
 {
         u8 memtype = (tile_flags & 0x0000ff00) >> 8;
         return likely((nvc0_pte_storage_type_map[memtype] != 0xff));
 }
 
-static int
+int
 nvc0_fb_init(struct nouveau_object *object)
 {
         struct nvc0_fb_priv *priv = (void *)object;
@@ -54,7 +48,7 @@ nvc0_fb_init(struct nouveau_object *object)
         return 0;
 }
 
-static void
+void
 nvc0_fb_dtor(struct nouveau_object *object)
 {
         struct nouveau_device *device = nv_device(object);
@@ -69,7 +63,7 @@ nvc0_fb_dtor(struct nouveau_object *object)
         nouveau_fb_destroy(&priv->base);
 }
 
-static int
+int
 nvc0_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
              struct nouveau_oclass *oclass, void *data, u32 size,
              struct nouveau_object **pobject)
@@ -78,13 +72,11 @@ nvc0_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nvc0_fb_priv *priv;
         int ret;
 
-        ret = nouveau_fb_create(parent, engine, oclass, &nvc0_ram_oclass, &priv);
+        ret = nouveau_fb_create(parent, engine, oclass, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
 
-        priv->base.memtype_valid = nvc0_fb_memtype_valid;
-
         priv->r100c10_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
         if (priv->r100c10_page) {
                 priv->r100c10 = pci_map_page(device->pdev, priv->r100c10_page,
@@ -97,14 +89,15 @@ nvc0_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-
-struct nouveau_oclass
-nvc0_fb_oclass = {
-        .handle = NV_SUBDEV(FB, 0xc0),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nvc0_fb_oclass = &(struct nouveau_fb_impl) {
+        .base.handle = NV_SUBDEV(FB, 0xc0),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nvc0_fb_ctor,
                 .dtor = nvc0_fb_dtor,
                 .init = nvc0_fb_init,
                 .fini = _nouveau_fb_fini,
         },
-};
+        .memtype = nvc0_fb_memtype_valid,
+        .ram = &nvc0_ram_oclass,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.h b/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.h
new file mode 100644
index 000000000000..9e1931eb746f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.h
@@ -0,0 +1,29 @@
+#ifndef __NVKM_RAM_NVC0_H__
+#define __NVKM_RAM_NVC0_H__
+
+#include "priv.h"
+#include "nv50.h"
+
+struct nvc0_fb_priv {
+        struct nouveau_fb base;
+        struct page *r100c10_page;
+        dma_addr_t r100c10;
+};
+
+int  nvc0_fb_ctor(struct nouveau_object *, struct nouveau_object *,
+                  struct nouveau_oclass *, void *, u32,
+                  struct nouveau_object **);
+void nvc0_fb_dtor(struct nouveau_object *);
+int  nvc0_fb_init(struct nouveau_object *);
+bool nvc0_fb_memtype_valid(struct nouveau_fb *, u32);
+
+
+#define nvc0_ram_create(p,e,o,d)                                               \
+        nvc0_ram_create_((p), (e), (o), sizeof(**d), (void **)d)
+int  nvc0_ram_create_(struct nouveau_object *, struct nouveau_object *,
+                      struct nouveau_oclass *, int, void **);
+int  nvc0_ram_get(struct nouveau_fb *, u64, u32, u32, u32,
+                  struct nouveau_mem **);
+void nvc0_ram_put(struct nouveau_fb *, struct nouveau_mem **);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nve0.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nve0.c
new file mode 100644
index 000000000000..595db50cfef3
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nve0.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nvc0.h"
+
+struct nouveau_oclass *
+nve0_fb_oclass = &(struct nouveau_fb_impl) {
+        .base.handle = NV_SUBDEV(FB, 0xe0),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nvc0_fb_ctor,
+                .dtor = nvc0_fb_dtor,
+                .init = nvc0_fb_init,
+                .fini = _nouveau_fb_fini,
+        },
+        .memtype = nvc0_fb_memtype_valid,
+        .ram = &nve0_ram_oclass,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h b/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h
index db9d6ddde52c..493125214e88 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h
@@ -12,6 +12,8 @@
 #define nouveau_ram_fini(p,s)                                                  \
         nouveau_object_fini(&(p)->base, (s))
 
+#define nouveau_ram_create_(p,e,o,s,d)                                         \
+        nouveau_object_create_((p), (e), (o), 0, (s), (void **)d)
 #define _nouveau_ram_dtor nouveau_object_destroy
 #define _nouveau_ram_init nouveau_object_init
 #define _nouveau_ram_fini nouveau_object_fini
@@ -26,10 +28,16 @@ extern struct nouveau_oclass nv44_ram_oclass;
 extern struct nouveau_oclass nv49_ram_oclass;
 extern struct nouveau_oclass nv4e_ram_oclass;
 extern struct nouveau_oclass nv50_ram_oclass;
+extern struct nouveau_oclass nva3_ram_oclass;
+extern struct nouveau_oclass nvaa_ram_oclass;
 extern struct nouveau_oclass nvc0_ram_oclass;
+extern struct nouveau_oclass nve0_ram_oclass;
 
-#define nouveau_fb_create(p,e,c,r,d)                                           \
-        nouveau_fb_create_((p), (e), (c), (r), sizeof(**d), (void **)d)
+int nouveau_sddr3_calc(struct nouveau_ram *ram);
+int nouveau_gddr5_calc(struct nouveau_ram *ram);
+
+#define nouveau_fb_create(p,e,c,d)                                             \
+        nouveau_fb_create_((p), (e), (c), sizeof(**d), (void **)d)
 #define nouveau_fb_destroy(p) ({                                               \
         struct nouveau_fb *pfb = (p);                                          \
         _nouveau_fb_dtor(nv_object(pfb));                                      \
@@ -44,44 +52,21 @@ extern struct nouveau_oclass nvc0_ram_oclass;
 })
 
 int nouveau_fb_create_(struct nouveau_object *, struct nouveau_object *,
-                       struct nouveau_oclass *, struct nouveau_oclass *,
-                       int length, void **pobject);
+                       struct nouveau_oclass *, int, void **);
 void _nouveau_fb_dtor(struct nouveau_object *);
 int  _nouveau_fb_init(struct nouveau_object *);
 int  _nouveau_fb_fini(struct nouveau_object *, bool);
 
-struct nouveau_bios;
-int  nouveau_fb_bios_memtype(struct nouveau_bios *);
+struct nouveau_fb_impl {
+        struct nouveau_oclass base;
+        struct nouveau_oclass *ram;
+        bool (*memtype)(struct nouveau_fb *, u32);
+};
 
 bool nv04_fb_memtype_valid(struct nouveau_fb *, u32 memtype);
+bool nv50_fb_memtype_valid(struct nouveau_fb *, u32 memtype);
 
-void nv10_fb_tile_init(struct nouveau_fb *, int i, u32 addr, u32 size,
-                       u32 pitch, u32 flags, struct nouveau_fb_tile *);
-void nv10_fb_tile_fini(struct nouveau_fb *, int i, struct nouveau_fb_tile *);
-void nv10_fb_tile_prog(struct nouveau_fb *, int, struct nouveau_fb_tile *);
-
-void nv20_fb_tile_init(struct nouveau_fb *, int i, u32 addr, u32 size,
-                       u32 pitch, u32 flags, struct nouveau_fb_tile *);
-void nv20_fb_tile_fini(struct nouveau_fb *, int i, struct nouveau_fb_tile *);
-void nv20_fb_tile_prog(struct nouveau_fb *, int, struct nouveau_fb_tile *);
-
-int  nv30_fb_init(struct nouveau_object *);
-void nv30_fb_tile_init(struct nouveau_fb *, int i, u32 addr, u32 size,
-                       u32 pitch, u32 flags, struct nouveau_fb_tile *);
-
-void nv40_fb_tile_comp(struct nouveau_fb *, int i, u32 size, u32 flags,
-                       struct nouveau_fb_tile *);
-
-int  nv41_fb_init(struct nouveau_object *);
-void nv41_fb_tile_prog(struct nouveau_fb *, int, struct nouveau_fb_tile *);
-
-int  nv44_fb_init(struct nouveau_object *);
-void nv44_fb_tile_prog(struct nouveau_fb *, int, struct nouveau_fb_tile *);
-
-void nv46_fb_tile_init(struct nouveau_fb *, int i, u32 addr, u32 size,
-                       u32 pitch, u32 flags, struct nouveau_fb_tile *);
-
-void __nv50_ram_put(struct nouveau_fb *, struct nouveau_mem *);
-extern int nv50_fb_memtype[0x80];
+struct nouveau_bios;
+int  nouveau_fb_bios_memtype(struct nouveau_bios *);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramfuc.h b/drivers/gpu/drm/nouveau/core/subdev/fb/ramfuc.h
new file mode 100644
index 000000000000..0f57fcfe0bbf
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramfuc.h
@@ -0,0 +1,118 @@
+#ifndef __NVKM_FBRAM_FUC_H__
+#define __NVKM_FBRAM_FUC_H__
+
+#include <subdev/pwr.h>
+
+struct ramfuc {
+        struct nouveau_memx *memx;
+        struct nouveau_fb *pfb;
+        int sequence;
+};
+
+struct ramfuc_reg {
+        int sequence;
+        bool force;
+        u32 addr[2];
+        u32 data;
+};
+
+static inline struct ramfuc_reg
+ramfuc_reg2(u32 addr1, u32 addr2)
+{
+        return (struct ramfuc_reg) {
+                .sequence = 0,
+                .addr = { addr1, addr2 },
+                .data = 0xdeadbeef,
+        };
+}
+
+static inline struct ramfuc_reg
+ramfuc_reg(u32 addr)
+{
+        return ramfuc_reg2(addr, addr);
+}
+
+static inline int
+ramfuc_init(struct ramfuc *ram, struct nouveau_fb *pfb)
+{
+        struct nouveau_pwr *ppwr = nouveau_pwr(pfb);
+        int ret;
+
+        ret = nouveau_memx_init(ppwr, &ram->memx);
+        if (ret)
+                return ret;
+
+        ram->sequence++;
+        ram->pfb = pfb;
+        return 0;
+}
+
+static inline int
+ramfuc_exec(struct ramfuc *ram, bool exec)
+{
+        int ret = 0;
+        if (ram->pfb) {
+                ret = nouveau_memx_fini(&ram->memx, exec);
+                ram->pfb = NULL;
+        }
+        return ret;
+}
+
+static inline u32
+ramfuc_rd32(struct ramfuc *ram, struct ramfuc_reg *reg)
+{
+        if (reg->sequence != ram->sequence)
+                reg->data = nv_rd32(ram->pfb, reg->addr[0]);
+        return reg->data;
+}
+
+static inline void
+ramfuc_wr32(struct ramfuc *ram, struct ramfuc_reg *reg, u32 data)
+{
+        reg->sequence = ram->sequence;
+        reg->data = data;
+        if (reg->addr[0] != reg->addr[1])
+                nouveau_memx_wr32(ram->memx, reg->addr[1], reg->data);
+        nouveau_memx_wr32(ram->memx, reg->addr[0], reg->data);
+}
+
+static inline void
+ramfuc_nuke(struct ramfuc *ram, struct ramfuc_reg *reg)
+{
+        reg->force = true;
+}
+
+static inline u32
+ramfuc_mask(struct ramfuc *ram, struct ramfuc_reg *reg, u32 mask, u32 data)
+{
+        u32 temp = ramfuc_rd32(ram, reg);
+        if (temp != ((temp & ~mask) | data) || reg->force) {
+                ramfuc_wr32(ram, reg, (temp & ~mask) | data);
+                reg->force = false;
+        }
+        return temp;
+}
+
+static inline void
+ramfuc_wait(struct ramfuc *ram, u32 addr, u32 mask, u32 data, u32 nsec)
+{
+        nouveau_memx_wait(ram->memx, addr, mask, data, nsec);
+}
+
+static inline void
+ramfuc_nsec(struct ramfuc *ram, u32 nsec)
+{
+        nouveau_memx_nsec(ram->memx, nsec);
+}
+
+#define ram_init(s,p)       ramfuc_init(&(s)->base, (p))
+#define ram_exec(s,e)       ramfuc_exec(&(s)->base, (e))
+#define ram_have(s,r)       ((s)->r_##r.addr != 0x000000)
+#define ram_rd32(s,r)       ramfuc_rd32(&(s)->base, &(s)->r_##r)
+#define ram_wr32(s,r,d)     ramfuc_wr32(&(s)->base, &(s)->r_##r, (d))
+#define ram_nuke(s,r)       ramfuc_nuke(&(s)->base, &(s)->r_##r)
+#define ram_mask(s,r,m,d)   ramfuc_mask(&(s)->base, &(s)->r_##r, (m), (d))
+#define ram_wait(s,r,m,d,n) ramfuc_wait(&(s)->base, (r), (m), (d), (n))
+#define ram_nsec(s,n)       ramfuc_nsec(&(s)->base, (n))
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv40.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv40.c
index ee49ac4dbdb6..7648beb11199 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv40.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv40.c
@@ -22,7 +22,154 @@
  * Authors: Ben Skeggs
  */
 
-#include "priv.h"
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/pll.h>
+#include <subdev/bios/init.h>
+#include <subdev/clock.h>
+#include <subdev/clock/pll.h>
+#include <subdev/timer.h>
+
+#include <engine/fifo.h>
+
+#include "nv40.h"
+
+int
+nv40_ram_calc(struct nouveau_fb *pfb, u32 freq)
+{
+        struct nouveau_bios *bios = nouveau_bios(pfb);
+        struct nv40_ram *ram = (void *)pfb->ram;
+        struct nvbios_pll pll;
+        int N1, M1, N2, M2;
+        int log2P, ret;
+
+        ret = nvbios_pll_parse(bios, 0x04, &pll);
+        if (ret) {
+                nv_error(pfb, "mclk pll data not found\n");
+                return ret;
+        }
+
+        ret = nv04_pll_calc(nv_subdev(pfb), &pll, freq,
+                            &N1, &M1, &N2, &M2, &log2P);
+        if (ret < 0)
+                return ret;
+
+        ram->ctrl  = 0x80000000 | (log2P << 16);
+        ram->ctrl |= min(pll.bias_p + log2P, (int)pll.max_p) << 20;
+        if (N2 == M2) {
+                ram->ctrl |= 0x00000100;
+                ram->coef  = (N1 << 8) | M1;
+        } else {
+                ram->ctrl |= 0x40000000;
+                ram->coef  = (N2 << 24) | (M2 << 16) | (N1 << 8) | M1;
+        }
+
+        return 0;
+}
+
+int
+nv40_ram_prog(struct nouveau_fb *pfb)
+{
+        struct nouveau_bios *bios = nouveau_bios(pfb);
+        struct nv40_ram *ram = (void *)pfb->ram;
+        struct bit_entry M;
+        u32 crtc_mask = 0;
+        u8  sr1[2];
+        int i;
+
+        /* determine which CRTCs are active, fetch VGA_SR1 for each */
+        for (i = 0; i < 2; i++) {
+                u32 vbl = nv_rd32(pfb, 0x600808 + (i * 0x2000));
+                u32 cnt = 0;
+                do {
+                        if (vbl != nv_rd32(pfb, 0x600808 + (i * 0x2000))) {
+                                nv_wr08(pfb, 0x0c03c4 + (i * 0x2000), 0x01);
+                                sr1[i] = nv_rd08(pfb, 0x0c03c5 + (i * 0x2000));
+                                if (!(sr1[i] & 0x20))
+                                        crtc_mask |= (1 << i);
+                                break;
+                        }
+                        udelay(1);
+                } while (cnt++ < 32);
+        }
+
+        /* wait for vblank start on active crtcs, disable memory access */
+        for (i = 0; i < 2; i++) {
+                if (!(crtc_mask & (1 << i)))
+                        continue;
+                nv_wait(pfb, 0x600808 + (i * 0x2000), 0x00010000, 0x00000000);
+                nv_wait(pfb, 0x600808 + (i * 0x2000), 0x00010000, 0x00010000);
+                nv_wr08(pfb, 0x0c03c4 + (i * 0x2000), 0x01);
+                nv_wr08(pfb, 0x0c03c5 + (i * 0x2000), sr1[i] | 0x20);
+        }
+
+        /* prepare ram for reclocking */
+        nv_wr32(pfb, 0x1002d4, 0x00000001); /* precharge */
+        nv_wr32(pfb, 0x1002d0, 0x00000001); /* refresh */
+        nv_wr32(pfb, 0x1002d0, 0x00000001); /* refresh */
+        nv_mask(pfb, 0x100210, 0x80000000, 0x00000000); /* no auto refresh */
+        nv_wr32(pfb, 0x1002dc, 0x00000001); /* enable self-refresh */
+
+        /* change the PLL of each memory partition */
+        nv_mask(pfb, 0x00c040, 0x0000c000, 0x00000000);
+        switch (nv_device(pfb)->chipset) {
+        case 0x40:
+        case 0x45:
+        case 0x41:
+        case 0x42:
+        case 0x47:
+                nv_mask(pfb, 0x004044, 0xc0771100, ram->ctrl);
+                nv_mask(pfb, 0x00402c, 0xc0771100, ram->ctrl);
+                nv_wr32(pfb, 0x004048, ram->coef);
+                nv_wr32(pfb, 0x004030, ram->coef);
+        case 0x43:
+        case 0x49:
+        case 0x4b:
+                nv_mask(pfb, 0x004038, 0xc0771100, ram->ctrl);
+                nv_wr32(pfb, 0x00403c, ram->coef);
+        default:
+                nv_mask(pfb, 0x004020, 0xc0771100, ram->ctrl);
+                nv_wr32(pfb, 0x004024, ram->coef);
+                break;
+        }
+        udelay(100);
+        nv_mask(pfb, 0x00c040, 0x0000c000, 0x0000c000);
+
+        /* re-enable normal operation of memory controller */
+        nv_wr32(pfb, 0x1002dc, 0x00000000);
+        nv_mask(pfb, 0x100210, 0x80000000, 0x80000000);
+        udelay(100);
+
+        /* execute memory reset script from vbios */
+        if (!bit_entry(bios, 'M', &M)) {
+                struct nvbios_init init = {
+                        .subdev = nv_subdev(pfb),
+                        .bios = bios,
+                        .offset = nv_ro16(bios, M.offset + 0x00),
+                        .execute = 1,
+                };
+
+                nvbios_exec(&init);
+        }
+
+        /* make sure we're in vblank (hopefully the same one as before), and
+         * then re-enable crtc memory access
+         */
+        for (i = 0; i < 2; i++) {
+                if (!(crtc_mask & (1 << i)))
+                        continue;
+                nv_wait(pfb, 0x600808 + (i * 0x2000), 0x00010000, 0x00010000);
+                nv_wr08(pfb, 0x0c03c4 + (i * 0x2000), 0x01);
+                nv_wr08(pfb, 0x0c03c5 + (i * 0x2000), sr1[i]);
+        }
+
+        return 0;
+}
+
+void
+nv40_ram_tidy(struct nouveau_fb *pfb)
+{
+}
 
 static int
 nv40_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
@@ -30,7 +177,7 @@ nv40_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
                 struct nouveau_object **pobject)
 {
         struct nouveau_fb *pfb = nouveau_fb(parent);
-        struct nouveau_ram *ram;
+        struct nv40_ram *ram;
         u32 pbus1218 = nv_rd32(pfb, 0x001218);
         int ret;
 
@@ -40,15 +187,18 @@ nv40_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
                 return ret;
 
         switch (pbus1218 & 0x00000300) {
-        case 0x00000000: ram->type = NV_MEM_TYPE_SDRAM; break;
-        case 0x00000100: ram->type = NV_MEM_TYPE_DDR1; break;
-        case 0x00000200: ram->type = NV_MEM_TYPE_GDDR3; break;
-        case 0x00000300: ram->type = NV_MEM_TYPE_DDR2; break;
+        case 0x00000000: ram->base.type = NV_MEM_TYPE_SDRAM; break;
+        case 0x00000100: ram->base.type = NV_MEM_TYPE_DDR1; break;
+        case 0x00000200: ram->base.type = NV_MEM_TYPE_GDDR3; break;
+        case 0x00000300: ram->base.type = NV_MEM_TYPE_DDR2; break;
         }
 
-        ram->size  =  nv_rd32(pfb, 0x10020c) & 0xff000000;
-        ram->parts = (nv_rd32(pfb, 0x100200) & 0x00000003) + 1;
-        ram->tags  =  nv_rd32(pfb, 0x100320);
+        ram->base.size  =  nv_rd32(pfb, 0x10020c) & 0xff000000;
+        ram->base.parts = (nv_rd32(pfb, 0x100200) & 0x00000003) + 1;
+        ram->base.tags  =  nv_rd32(pfb, 0x100320);
+        ram->base.calc = nv40_ram_calc;
+        ram->base.prog = nv40_ram_prog;
+        ram->base.tidy = nv40_ram_tidy;
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv41.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv41.c
index 1dab7e12abab..d64498a4d9ee 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv41.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv41.c
@@ -22,7 +22,7 @@
  * Authors: Ben Skeggs
  */
 
-#include "priv.h"
+#include "nv40.h"
 
 static int
 nv41_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
@@ -30,7 +30,7 @@ nv41_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
                 struct nouveau_object **pobject)
 {
         struct nouveau_fb *pfb = nouveau_fb(parent);
-        struct nouveau_ram *ram;
+        struct nv40_ram *ram;
         u32 pfb474 = nv_rd32(pfb, 0x100474);
         int ret;
 
@@ -40,15 +40,18 @@ nv41_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
                 return ret;
 
         if (pfb474 & 0x00000004)
-                ram->type = NV_MEM_TYPE_GDDR3;
+                ram->base.type = NV_MEM_TYPE_GDDR3;
         if (pfb474 & 0x00000002)
-                ram->type = NV_MEM_TYPE_DDR2;
+                ram->base.type = NV_MEM_TYPE_DDR2;
         if (pfb474 & 0x00000001)
-                ram->type = NV_MEM_TYPE_DDR1;
+                ram->base.type = NV_MEM_TYPE_DDR1;
 
-        ram->size  =  nv_rd32(pfb, 0x10020c) & 0xff000000;
-        ram->parts = (nv_rd32(pfb, 0x100200) & 0x00000003) + 1;
-        ram->tags  =  nv_rd32(pfb, 0x100320);
+        ram->base.size  =  nv_rd32(pfb, 0x10020c) & 0xff000000;
+        ram->base.parts = (nv_rd32(pfb, 0x100200) & 0x00000003) + 1;
+        ram->base.tags  =  nv_rd32(pfb, 0x100320);
+        ram->base.calc = nv40_ram_calc;
+        ram->base.prog = nv40_ram_prog;
+        ram->base.tidy = nv40_ram_tidy;
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv44.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv44.c
index 25fff842e5c1..089acac810c5 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv44.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv44.c
@@ -22,7 +22,7 @@
  * Authors: Ben Skeggs
  */
 
-#include "priv.h"
+#include "nv40.h"
 
 static int
 nv44_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
@@ -30,7 +30,7 @@ nv44_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
                 struct nouveau_object **pobject)
 {
         struct nouveau_fb *pfb = nouveau_fb(parent);
-        struct nouveau_ram *ram;
+        struct nv40_ram *ram;
         u32 pfb474 = nv_rd32(pfb, 0x100474);
         int ret;
 
@@ -40,13 +40,16 @@ nv44_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
                 return ret;
 
         if (pfb474 & 0x00000004)
-                ram->type = NV_MEM_TYPE_GDDR3;
+                ram->base.type = NV_MEM_TYPE_GDDR3;
         if (pfb474 & 0x00000002)
-                ram->type = NV_MEM_TYPE_DDR2;
+                ram->base.type = NV_MEM_TYPE_DDR2;
         if (pfb474 & 0x00000001)
-                ram->type = NV_MEM_TYPE_DDR1;
+                ram->base.type = NV_MEM_TYPE_DDR1;
 
-        ram->size = nv_rd32(pfb, 0x10020c) & 0xff000000;
+        ram->base.size = nv_rd32(pfb, 0x10020c) & 0xff000000;
+        ram->base.calc = nv40_ram_calc;
+        ram->base.prog = nv40_ram_prog;
+        ram->base.tidy = nv40_ram_tidy;
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv49.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv49.c
index ab7ef0ac9e34..baa013afa57b 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv49.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv49.c
@@ -22,7 +22,7 @@
  * Authors: Ben Skeggs
  */
 
-#include "priv.h"
+#include "nv40.h"
 
 static int
 nv49_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
@@ -30,7 +30,7 @@ nv49_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
                 struct nouveau_object **pobject)
 {
         struct nouveau_fb *pfb = nouveau_fb(parent);
-        struct nouveau_ram *ram;
+        struct nv40_ram *ram;
         u32 pfb914 = nv_rd32(pfb, 0x100914);
         int ret;
 
@@ -40,15 +40,18 @@ nv49_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
                 return ret;
 
         switch (pfb914 & 0x00000003) {
-        case 0x00000000: ram->type = NV_MEM_TYPE_DDR1; break;
-        case 0x00000001: ram->type = NV_MEM_TYPE_DDR2; break;
-        case 0x00000002: ram->type = NV_MEM_TYPE_GDDR3; break;
+        case 0x00000000: ram->base.type = NV_MEM_TYPE_DDR1; break;
+        case 0x00000001: ram->base.type = NV_MEM_TYPE_DDR2; break;
+        case 0x00000002: ram->base.type = NV_MEM_TYPE_GDDR3; break;
         case 0x00000003: break;
         }
 
-        ram->size  =  nv_rd32(pfb, 0x10020c) & 0xff000000;
-        ram->parts = (nv_rd32(pfb, 0x100200) & 0x00000003) + 1;
-        ram->tags  =  nv_rd32(pfb, 0x100320);
+        ram->base.size  =  nv_rd32(pfb, 0x10020c) & 0xff000000;
+        ram->base.parts = (nv_rd32(pfb, 0x100200) & 0x00000003) + 1;
+        ram->base.tags  =  nv_rd32(pfb, 0x100320);
+        ram->base.calc = nv40_ram_calc;
+        ram->base.prog = nv40_ram_prog;
+        ram->base.tidy = nv40_ram_tidy;
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv50.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv50.c
index 903baff77fdd..76762a17d89c 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv50.c
@@ -23,8 +23,215 @@
  */
 
 #include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/pll.h>
+#include <subdev/bios/perf.h>
+#include <subdev/bios/timing.h>
+#include <subdev/clock/pll.h>
+#include <subdev/fb.h>
+
+#include <core/option.h>
 #include <core/mm.h>
-#include "priv.h"
+
+#include "ramseq.h"
+
+#include "nv50.h"
+
+struct nv50_ramseq {
+        struct hwsq base;
+        struct hwsq_reg r_0x002504;
+        struct hwsq_reg r_0x004008;
+        struct hwsq_reg r_0x00400c;
+        struct hwsq_reg r_0x00c040;
+        struct hwsq_reg r_0x100210;
+        struct hwsq_reg r_0x1002d0;
+        struct hwsq_reg r_0x1002d4;
+        struct hwsq_reg r_0x1002dc;
+        struct hwsq_reg r_0x100da0[8];
+        struct hwsq_reg r_0x100e20;
+        struct hwsq_reg r_0x100e24;
+        struct hwsq_reg r_0x611200;
+        struct hwsq_reg r_timing[9];
+        struct hwsq_reg r_mr[4];
+};
+
+struct nv50_ram {
+        struct nouveau_ram base;
+        struct nv50_ramseq hwsq;
+};
+
+#define QFX5800NVA0 1
+
+static int
+nv50_ram_calc(struct nouveau_fb *pfb, u32 freq)
+{
+        struct nouveau_bios *bios = nouveau_bios(pfb);
+        struct nv50_ram *ram = (void *)pfb->ram;
+        struct nv50_ramseq *hwsq = &ram->hwsq;
+        struct nvbios_perfE perfE;
+        struct nvbios_pll mpll;
+        struct bit_entry M;
+        struct {
+                u32 data;
+                u8  size;
+        } ramcfg, timing;
+        u8  ver, hdr, cnt, strap;
+        u32 data;
+        int N1, M1, N2, M2, P;
+        int ret, i;
+
+        /* lookup closest matching performance table entry for frequency */
+        i = 0;
+        do {
+                ramcfg.data = nvbios_perfEp(bios, i++, &ver, &hdr, &cnt,
+                                           &ramcfg.size, &perfE);
+                if (!ramcfg.data || (ver < 0x25 || ver >= 0x40) ||
+                    (ramcfg.size < 2)) {
+                        nv_error(pfb, "invalid/missing perftab entry\n");
+                        return -EINVAL;
+                }
+        } while (perfE.memory < freq);
+
+        /* locate specific data set for the attached memory */
+        if (bit_entry(bios, 'M', &M) || M.version != 1 || M.length < 5) {
+                nv_error(pfb, "invalid/missing memory table\n");
+                return -EINVAL;
+        }
+
+        strap = (nv_rd32(pfb, 0x101000) & 0x0000003c) >> 2;
+        data = nv_ro16(bios, M.offset + 3);
+        if (data)
+                strap = nv_ro08(bios, data + strap);
+
+        if (strap >= cnt) {
+                nv_error(pfb, "invalid ramcfg strap\n");
+                return -EINVAL;
+        }
+
+        ramcfg.data += hdr + (strap * ramcfg.size);
+
+        /* lookup memory timings, if bios says they're present */
+        strap = nv_ro08(bios, ramcfg.data + 0x01);
+        if (strap != 0xff) {
+                timing.data = nvbios_timing_entry(bios, strap, &ver, &hdr);
+                if (!timing.data || ver != 0x10 || hdr < 0x12) {
+                        nv_error(pfb, "invalid/missing timing entry "
+                                 "%02x %04x %02x %02x\n",
+                                 strap, timing.data, ver, hdr);
+                        return -EINVAL;
+                }
+        } else {
+                timing.data = 0;
+        }
+
+        ret = ram_init(hwsq, nv_subdev(pfb));
+        if (ret)
+                return ret;
+
+        ram_wait(hwsq, 0x01, 0x00); /* wait for !vblank */
+        ram_wait(hwsq, 0x01, 0x01); /* wait for vblank */
+        ram_wr32(hwsq, 0x611200, 0x00003300);
+        ram_wr32(hwsq, 0x002504, 0x00000001); /* block fifo */
+        ram_nsec(hwsq, 8000);
+        ram_setf(hwsq, 0x10, 0x00); /* disable fb */
+        ram_wait(hwsq, 0x00, 0x01); /* wait for fb disabled */
+
+        ram_wr32(hwsq, 0x1002d4, 0x00000001); /* precharge */
+        ram_wr32(hwsq, 0x1002d0, 0x00000001); /* refresh */
+        ram_wr32(hwsq, 0x1002d0, 0x00000001); /* refresh */
+        ram_wr32(hwsq, 0x100210, 0x00000000); /* disable auto-refresh */
+        ram_wr32(hwsq, 0x1002dc, 0x00000001); /* enable self-refresh */
+
+        ret = nvbios_pll_parse(bios, 0x004008, &mpll);
+        mpll.vco2.max_freq = 0;
+        if (ret == 0) {
+                ret = nv04_pll_calc(nv_subdev(pfb), &mpll, freq,
+                                   &N1, &M1, &N2, &M2, &P);
+                if (ret == 0)
+                        ret = -EINVAL;
+        }
+
+        if (ret < 0)
+                return ret;
+
+        ram_mask(hwsq, 0x00c040, 0xc000c000, 0x0000c000);
+        ram_mask(hwsq, 0x004008, 0x00000200, 0x00000200);
+        ram_mask(hwsq, 0x00400c, 0x0000ffff, (N1 << 8) | M1);
+        ram_mask(hwsq, 0x004008, 0x81ff0000, 0x80000000 | (mpll.bias_p << 19) |
+                                             (P << 22) | (P << 16));
+#if QFX5800NVA0
+        for (i = 0; i < 8; i++)
+                ram_mask(hwsq, 0x100da0[i], 0x00000000, 0x00000000); /*XXX*/
+#endif
+        ram_nsec(hwsq, 96000); /*XXX*/
+        ram_mask(hwsq, 0x004008, 0x00002200, 0x00002000);
+
+        ram_wr32(hwsq, 0x1002dc, 0x00000000); /* disable self-refresh */
+        ram_wr32(hwsq, 0x100210, 0x80000000); /* enable auto-refresh */
+
+        ram_nsec(hwsq, 12000);
+
+        switch (ram->base.type) {
+        case NV_MEM_TYPE_DDR2:
+                ram_nuke(hwsq, mr[0]); /* force update */
+                ram_mask(hwsq, mr[0], 0x000, 0x000);
+                break;
+        case NV_MEM_TYPE_GDDR3:
+                ram_mask(hwsq, mr[2], 0x000, 0x000);
+                ram_nuke(hwsq, mr[0]); /* force update */
+                ram_mask(hwsq, mr[0], 0x000, 0x000);
+                break;
+        default:
+                break;
+        }
+
+        ram_mask(hwsq, timing[3], 0x00000000, 0x00000000); /*XXX*/
+        ram_mask(hwsq, timing[1], 0x00000000, 0x00000000); /*XXX*/
+        ram_mask(hwsq, timing[6], 0x00000000, 0x00000000); /*XXX*/
+        ram_mask(hwsq, timing[7], 0x00000000, 0x00000000); /*XXX*/
+        ram_mask(hwsq, timing[8], 0x00000000, 0x00000000); /*XXX*/
+        ram_mask(hwsq, timing[0], 0x00000000, 0x00000000); /*XXX*/
+        ram_mask(hwsq, timing[2], 0x00000000, 0x00000000); /*XXX*/
+        ram_mask(hwsq, timing[4], 0x00000000, 0x00000000); /*XXX*/
+        ram_mask(hwsq, timing[5], 0x00000000, 0x00000000); /*XXX*/
+
+        ram_mask(hwsq, timing[0], 0x00000000, 0x00000000); /*XXX*/
+
+#if QFX5800NVA0
+        ram_nuke(hwsq, 0x100e24);
+        ram_mask(hwsq, 0x100e24, 0x00000000, 0x00000000);
+        ram_nuke(hwsq, 0x100e20);
+        ram_mask(hwsq, 0x100e20, 0x00000000, 0x00000000);
+#endif
+
+        ram_mask(hwsq, mr[0], 0x100, 0x100);
+        ram_mask(hwsq, mr[0], 0x100, 0x000);
+
+        ram_setf(hwsq, 0x10, 0x01); /* enable fb */
+        ram_wait(hwsq, 0x00, 0x00); /* wait for fb enabled */
+        ram_wr32(hwsq, 0x611200, 0x00003330);
+        ram_wr32(hwsq, 0x002504, 0x00000000); /* un-block fifo */
+        return 0;
+}
+
+static int
+nv50_ram_prog(struct nouveau_fb *pfb)
+{
+        struct nouveau_device *device = nv_device(pfb);
+        struct nv50_ram *ram = (void *)pfb->ram;
+        struct nv50_ramseq *hwsq = &ram->hwsq;
+
+        ram_exec(hwsq, nouveau_boolopt(device->cfgopt, "NvMemExec", false));
+        return 0;
+}
+
+static void
+nv50_ram_tidy(struct nouveau_fb *pfb)
+{
+        struct nv50_ram *ram = (void *)pfb->ram;
+        struct nv50_ramseq *hwsq = &ram->hwsq;
+        ram_exec(hwsq, false);
+}
 
 void
 __nv50_ram_put(struct nouveau_fb *pfb, struct nouveau_mem *mem)
@@ -57,7 +264,7 @@ nv50_ram_put(struct nouveau_fb *pfb, struct nouveau_mem **pmem)
         kfree(mem);
 }
 
-static int
+int
 nv50_ram_get(struct nouveau_fb *pfb, u64 size, u32 align, u32 ncmin,
              u32 memtype, struct nouveau_mem **pmem)
 {
@@ -160,77 +367,114 @@ nv50_fb_vram_rblock(struct nouveau_fb *pfb, struct nouveau_ram *ram)
         return rblock_size;
 }
 
-static int
-nv50_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
-                struct nouveau_oclass *oclass, void *data, u32 datasize,
-                struct nouveau_object **pobject)
+int
+nv50_ram_create_(struct nouveau_object *parent, struct nouveau_object *engine,
+                 struct nouveau_oclass *oclass, int length, void **pobject)
 {
-        struct nouveau_fb *pfb = nouveau_fb(parent);
-        struct nouveau_device *device = nv_device(pfb);
-        struct nouveau_bios *bios = nouveau_bios(device);
-        struct nouveau_ram *ram;
         const u32 rsvd_head = ( 256 * 1024) >> 12; /* vga memory */
         const u32 rsvd_tail = (1024 * 1024) >> 12; /* vbios etc */
-        u32 size;
+        struct nouveau_bios *bios = nouveau_bios(parent);
+        struct nouveau_fb *pfb = nouveau_fb(parent);
+        struct nouveau_ram *ram;
         int ret;
 
-        ret = nouveau_ram_create(parent, engine, oclass, &ram);
-        *pobject = nv_object(ram);
+        ret = nouveau_ram_create_(parent, engine, oclass, length, pobject);
+        ram = *pobject;
         if (ret)
                 return ret;
 
         ram->size = nv_rd32(pfb, 0x10020c);
-        ram->size = (ram->size & 0xffffff00) |
-                       ((ram->size & 0x000000ff) << 32);
-
-        size = (ram->size >> 12) - rsvd_head - rsvd_tail;
-        switch (device->chipset) {
-        case 0xaa:
-        case 0xac:
-        case 0xaf: /* IGPs, no reordering, no real VRAM */
-                ret = nouveau_mm_init(&pfb->vram, rsvd_head, size, 1);
-                if (ret)
-                        return ret;
+        ram->size = (ram->size & 0xffffff00) | ((ram->size & 0x000000ff) << 32);
 
-                ram->type   = NV_MEM_TYPE_STOLEN;
-                ram->stolen = (u64)nv_rd32(pfb, 0x100e10) << 12;
+        switch (nv_rd32(pfb, 0x100714) & 0x00000007) {
+        case 0: ram->type = NV_MEM_TYPE_DDR1; break;
+        case 1:
+                if (nouveau_fb_bios_memtype(bios) == NV_MEM_TYPE_DDR3)
+                        ram->type = NV_MEM_TYPE_DDR3;
+                else
+                        ram->type = NV_MEM_TYPE_DDR2;
                 break;
+        case 2: ram->type = NV_MEM_TYPE_GDDR3; break;
+        case 3: ram->type = NV_MEM_TYPE_GDDR4; break;
+        case 4: ram->type = NV_MEM_TYPE_GDDR5; break;
         default:
-                switch (nv_rd32(pfb, 0x100714) & 0x00000007) {
-                case 0: ram->type = NV_MEM_TYPE_DDR1; break;
-                case 1:
-                        if (nouveau_fb_bios_memtype(bios) == NV_MEM_TYPE_DDR3)
-                                ram->type = NV_MEM_TYPE_DDR3;
-                        else
-                                ram->type = NV_MEM_TYPE_DDR2;
-                        break;
-                case 2: ram->type = NV_MEM_TYPE_GDDR3; break;
-                case 3: ram->type = NV_MEM_TYPE_GDDR4; break;
-                case 4: ram->type = NV_MEM_TYPE_GDDR5; break;
-                default:
-                        break;
-                }
-
-                ret = nouveau_mm_init(&pfb->vram, rsvd_head, size,
-                                      nv50_fb_vram_rblock(pfb, ram) >> 12);
-                if (ret)
-                        return ret;
-
-                ram->ranks = (nv_rd32(pfb, 0x100200) & 0x4) ? 2 : 1;
-                ram->tags  =  nv_rd32(pfb, 0x100320);
                 break;
         }
 
+        ret = nouveau_mm_init(&pfb->vram, rsvd_head, (ram->size >> 12) -
+                              (rsvd_head + rsvd_tail),
+                              nv50_fb_vram_rblock(pfb, ram) >> 12);
+        if (ret)
+                return ret;
+
+        ram->ranks = (nv_rd32(pfb, 0x100200) & 0x4) ? 2 : 1;
+        ram->tags  =  nv_rd32(pfb, 0x100320);
         ram->get = nv50_ram_get;
         ram->put = nv50_ram_put;
         return 0;
 }
 
+static int
+nv50_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+              struct nouveau_oclass *oclass, void *data, u32 datasize,
+              struct nouveau_object **pobject)
+{
+        struct nv50_ram *ram;
+        int ret, i;
+
+        ret = nv50_ram_create(parent, engine, oclass, &ram);
+        *pobject = nv_object(ram);
+        if (ret)
+                return ret;
+
+        switch (ram->base.type) {
+        case NV_MEM_TYPE_DDR2:
+        case NV_MEM_TYPE_GDDR3:
+                ram->base.calc = nv50_ram_calc;
+                ram->base.prog = nv50_ram_prog;
+                ram->base.tidy = nv50_ram_tidy;
+                break;
+        default:
+                nv_warn(ram, "reclocking of this ram type unsupported\n");
+                return 0;
+        }
+
+        ram->hwsq.r_0x002504 = hwsq_reg(0x002504);
+        ram->hwsq.r_0x00c040 = hwsq_reg(0x00c040);
+        ram->hwsq.r_0x004008 = hwsq_reg(0x004008);
+        ram->hwsq.r_0x00400c = hwsq_reg(0x00400c);
+        ram->hwsq.r_0x100210 = hwsq_reg(0x100210);
+        ram->hwsq.r_0x1002d0 = hwsq_reg(0x1002d0);
+        ram->hwsq.r_0x1002d4 = hwsq_reg(0x1002d4);
+        ram->hwsq.r_0x1002dc = hwsq_reg(0x1002dc);
+        for (i = 0; i < 8; i++)
+                ram->hwsq.r_0x100da0[i] = hwsq_reg(0x100da0 + (i * 0x04));
+        ram->hwsq.r_0x100e20 = hwsq_reg(0x100e20);
+        ram->hwsq.r_0x100e24 = hwsq_reg(0x100e24);
+        ram->hwsq.r_0x611200 = hwsq_reg(0x611200);
+
+        for (i = 0; i < 9; i++)
+                ram->hwsq.r_timing[i] = hwsq_reg(0x100220 + (i * 0x04));
+
+        if (ram->base.ranks > 1) {
+                ram->hwsq.r_mr[0] = hwsq_reg2(0x1002c0, 0x1002c8);
+                ram->hwsq.r_mr[1] = hwsq_reg2(0x1002c4, 0x1002cc);
+                ram->hwsq.r_mr[2] = hwsq_reg2(0x1002e0, 0x1002e8);
+                ram->hwsq.r_mr[3] = hwsq_reg2(0x1002e4, 0x1002ec);
+        } else {
+                ram->hwsq.r_mr[0] = hwsq_reg(0x1002c0);
+                ram->hwsq.r_mr[1] = hwsq_reg(0x1002c4);
+                ram->hwsq.r_mr[2] = hwsq_reg(0x1002e0);
+                ram->hwsq.r_mr[3] = hwsq_reg(0x1002e4);
+        }
+
+        return 0;
+}
+
 struct nouveau_oclass
 nv50_ram_oclass = {
-        .handle = 0,
         .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv50_ram_create,
+                .ctor = nv50_ram_ctor,
                 .dtor = _nouveau_ram_dtor,
                 .init = _nouveau_ram_init,
                 .fini = _nouveau_ram_fini,
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c
new file mode 100644
index 000000000000..f6292cd9207c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c
@@ -0,0 +1,447 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/pll.h>
+#include <subdev/bios/rammap.h>
+#include <subdev/bios/timing.h>
+
+#include <subdev/clock/nva3.h>
+#include <subdev/clock/pll.h>
+
+#include <core/option.h>
+
+#include "ramfuc.h"
+
+#include "nv50.h"
+
+struct nva3_ramfuc {
+        struct ramfuc base;
+        struct ramfuc_reg r_0x004000;
+        struct ramfuc_reg r_0x004004;
+        struct ramfuc_reg r_0x004018;
+        struct ramfuc_reg r_0x004128;
+        struct ramfuc_reg r_0x004168;
+        struct ramfuc_reg r_0x100200;
+        struct ramfuc_reg r_0x100210;
+        struct ramfuc_reg r_0x100220[9];
+        struct ramfuc_reg r_0x1002d0;
+        struct ramfuc_reg r_0x1002d4;
+        struct ramfuc_reg r_0x1002dc;
+        struct ramfuc_reg r_0x10053c;
+        struct ramfuc_reg r_0x1005a0;
+        struct ramfuc_reg r_0x1005a4;
+        struct ramfuc_reg r_0x100714;
+        struct ramfuc_reg r_0x100718;
+        struct ramfuc_reg r_0x10071c;
+        struct ramfuc_reg r_0x100760;
+        struct ramfuc_reg r_0x1007a0;
+        struct ramfuc_reg r_0x1007e0;
+        struct ramfuc_reg r_0x10f804;
+        struct ramfuc_reg r_0x1110e0;
+        struct ramfuc_reg r_0x111100;
+        struct ramfuc_reg r_0x111104;
+        struct ramfuc_reg r_0x611200;
+        struct ramfuc_reg r_mr[4];
+};
+
+struct nva3_ram {
+        struct nouveau_ram base;
+        struct nva3_ramfuc fuc;
+};
+
+static int
+nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
+{
+        struct nouveau_bios *bios = nouveau_bios(pfb);
+        struct nva3_ram *ram = (void *)pfb->ram;
+        struct nva3_ramfuc *fuc = &ram->fuc;
+        struct nva3_clock_info mclk;
+        struct bit_entry M;
+        u8  ver, cnt, strap;
+        u32 data;
+        struct {
+                u32 data;
+                u8  size;
+        } rammap, ramcfg, timing;
+        u32 r004018, r100760, ctrl;
+        u32 unk714, unk718, unk71c;
+        int ret;
+
+        /* lookup memory config data relevant to the target frequency */
+        rammap.data = nvbios_rammap_match(bios, freq / 1000, &ver, &rammap.size,
+                                         &cnt, &ramcfg.size);
+        if (!rammap.data || ver != 0x10 || rammap.size < 0x0e) {
+                nv_error(pfb, "invalid/missing rammap entry\n");
+                return -EINVAL;
+        }
+
+        /* locate specific data set for the attached memory */
+        if (bit_entry(bios, 'M', &M) || M.version != 2 || M.length < 3) {
+                nv_error(pfb, "invalid/missing memory table\n");
+                return -EINVAL;
+        }
+
+        strap = (nv_rd32(pfb, 0x101000) & 0x0000003c) >> 2;
+        data = nv_ro16(bios, M.offset + 1);
+        if (data)
+                strap = nv_ro08(bios, data + strap);
+
+        if (strap >= cnt) {
+                nv_error(pfb, "invalid ramcfg strap\n");
+                return -EINVAL;
+        }
+
+        ramcfg.data = rammap.data + rammap.size + (strap * ramcfg.size);
+        if (!ramcfg.data || ver != 0x10 || ramcfg.size < 0x0e) {
+                nv_error(pfb, "invalid/missing ramcfg entry\n");
+                return -EINVAL;
+        }
+
+        /* lookup memory timings, if bios says they're present */
+        strap = nv_ro08(bios, ramcfg.data + 0x01);
+        if (strap != 0xff) {
+                timing.data = nvbios_timing_entry(bios, strap, &ver,
+                                                 &timing.size);
+                if (!timing.data || ver != 0x10 || timing.size < 0x19) {
+                        nv_error(pfb, "invalid/missing timing entry\n");
+                        return -EINVAL;
+                }
+        } else {
+                timing.data = 0;
+        }
+
+        ret = nva3_clock_info(nouveau_clock(pfb), 0x12, 0x4000, freq, &mclk);
+        if (ret < 0) {
+                nv_error(pfb, "failed mclk calculation\n");
+                return ret;
+        }
+
+        ret = ram_init(fuc, pfb);
+        if (ret)
+                return ret;
+
+        /* XXX: where the fuck does 750MHz come from? */
+        if (freq <= 750000) {
+                r004018 = 0x10000000;
+                r100760 = 0x22222222;
+        } else {
+                r004018 = 0x00000000;
+                r100760 = 0x00000000;
+        }
+
+        ctrl = ram_rd32(fuc, 0x004000);
+        if (ctrl & 0x00000008) {
+                if (mclk.pll) {
+                        ram_mask(fuc, 0x004128, 0x00000101, 0x00000101);
+                        ram_wr32(fuc, 0x004004, mclk.pll);
+                        ram_wr32(fuc, 0x004000, (ctrl |= 0x00000001));
+                        ram_wr32(fuc, 0x004000, (ctrl &= 0xffffffef));
+                        ram_wait(fuc, 0x004000, 0x00020000, 0x00020000, 64000);
+                        ram_wr32(fuc, 0x004000, (ctrl |= 0x00000010));
+                        ram_wr32(fuc, 0x004018, 0x00005000 | r004018);
+                        ram_wr32(fuc, 0x004000, (ctrl |= 0x00000004));
+                }
+        } else {
+                u32 ssel = 0x00000101;
+                if (mclk.clk)
+                        ssel |= mclk.clk;
+                else
+                        ssel |= 0x00080000; /* 324MHz, shouldn't matter... */
+                ram_mask(fuc, 0x004168, 0x003f3141, ctrl);
+        }
+
+        if ( (nv_ro08(bios, ramcfg.data + 0x02) & 0x10)) {
+                ram_mask(fuc, 0x111104, 0x00000600, 0x00000000);
+        } else {
+                ram_mask(fuc, 0x111100, 0x40000000, 0x40000000);
+                ram_mask(fuc, 0x111104, 0x00000180, 0x00000000);
+        }
+
+        if (!(nv_ro08(bios, rammap.data + 0x04) & 0x02))
+                ram_mask(fuc, 0x100200, 0x00000800, 0x00000000);
+        ram_wr32(fuc, 0x611200, 0x00003300);
+        if (!(nv_ro08(bios, ramcfg.data + 0x02) & 0x10))
+                ram_wr32(fuc, 0x111100, 0x4c020000); /*XXX*/
+
+        ram_wr32(fuc, 0x1002d4, 0x00000001);
+        ram_wr32(fuc, 0x1002d0, 0x00000001);
+        ram_wr32(fuc, 0x1002d0, 0x00000001);
+        ram_wr32(fuc, 0x100210, 0x00000000);
+        ram_wr32(fuc, 0x1002dc, 0x00000001);
+        ram_nsec(fuc, 2000);
+
+        ctrl = ram_rd32(fuc, 0x004000);
+        if (!(ctrl & 0x00000008) && mclk.pll) {
+                ram_wr32(fuc, 0x004000, (ctrl |=  0x00000008));
+                ram_mask(fuc, 0x1110e0, 0x00088000, 0x00088000);
+                ram_wr32(fuc, 0x004018, 0x00001000);
+                ram_wr32(fuc, 0x004000, (ctrl &= ~0x00000001));
+                ram_wr32(fuc, 0x004004, mclk.pll);
+                ram_wr32(fuc, 0x004000, (ctrl |=  0x00000001));
+                udelay(64);
+                ram_wr32(fuc, 0x004018, 0x00005000 | r004018);
+                udelay(20);
+        } else
+        if (!mclk.pll) {
+                ram_mask(fuc, 0x004168, 0x003f3040, mclk.clk);
+                ram_wr32(fuc, 0x004000, (ctrl |= 0x00000008));
+                ram_mask(fuc, 0x1110e0, 0x00088000, 0x00088000);
+                ram_wr32(fuc, 0x004018, 0x0000d000 | r004018);
+        }
+
+        if ( (nv_ro08(bios, rammap.data + 0x04) & 0x08)) {
+                u32 unk5a0 = (nv_ro16(bios, ramcfg.data + 0x05) << 8) |
+                              nv_ro08(bios, ramcfg.data + 0x05);
+                u32 unk5a4 = (nv_ro16(bios, ramcfg.data + 0x07));
+                u32 unk804 = (nv_ro08(bios, ramcfg.data + 0x09) & 0xf0) << 16 |
+                             (nv_ro08(bios, ramcfg.data + 0x03) & 0x0f) << 16 |
+                             (nv_ro08(bios, ramcfg.data + 0x09) & 0x0f) |
+                             0x80000000;
+                ram_wr32(fuc, 0x1005a0, unk5a0);
+                ram_wr32(fuc, 0x1005a4, unk5a4);
+                ram_wr32(fuc, 0x10f804, unk804);
+                ram_mask(fuc, 0x10053c, 0x00001000, 0x00000000);
+        } else {
+                ram_mask(fuc, 0x10053c, 0x00001000, 0x00001000);
+                ram_mask(fuc, 0x10f804, 0x80000000, 0x00000000);
+                ram_mask(fuc, 0x100760, 0x22222222, r100760);
+                ram_mask(fuc, 0x1007a0, 0x22222222, r100760);
+                ram_mask(fuc, 0x1007e0, 0x22222222, r100760);
+        }
+
+        if (mclk.pll) {
+                ram_mask(fuc, 0x1110e0, 0x00088000, 0x00011000);
+                ram_wr32(fuc, 0x004000, (ctrl &= ~0x00000008));
+        }
+
+        /*XXX: LEAVE */
+        ram_wr32(fuc, 0x1002dc, 0x00000000);
+        ram_wr32(fuc, 0x1002d4, 0x00000001);
+        ram_wr32(fuc, 0x100210, 0x80000000);
+        ram_nsec(fuc, 1000);
+        ram_nsec(fuc, 1000);
+
+        ram_mask(fuc, mr[2], 0x00000000, 0x00000000);
+        ram_nsec(fuc, 1000);
+        ram_nuke(fuc, mr[0]);
+        ram_mask(fuc, mr[0], 0x00000000, 0x00000000);
+        ram_nsec(fuc, 1000);
+
+        ram_mask(fuc, 0x100220[3], 0x00000000, 0x00000000);
+        ram_mask(fuc, 0x100220[1], 0x00000000, 0x00000000);
+        ram_mask(fuc, 0x100220[6], 0x00000000, 0x00000000);
+        ram_mask(fuc, 0x100220[7], 0x00000000, 0x00000000);
+        ram_mask(fuc, 0x100220[2], 0x00000000, 0x00000000);
+        ram_mask(fuc, 0x100220[4], 0x00000000, 0x00000000);
+        ram_mask(fuc, 0x100220[5], 0x00000000, 0x00000000);
+        ram_mask(fuc, 0x100220[0], 0x00000000, 0x00000000);
+        ram_mask(fuc, 0x100220[8], 0x00000000, 0x00000000);
+
+        data = (nv_ro08(bios, ramcfg.data + 0x02) & 0x08) ? 0x00000000 : 0x00001000;
+        ram_mask(fuc, 0x100200, 0x00001000, data);
+
+        unk714 = ram_rd32(fuc, 0x100714) & ~0xf0000010;
+        unk718 = ram_rd32(fuc, 0x100718) & ~0x00000100;
+        unk71c = ram_rd32(fuc, 0x10071c) & ~0x00000100;
+        if ( (nv_ro08(bios, ramcfg.data + 0x02) & 0x20))
+                unk714 |= 0xf0000000;
+        if (!(nv_ro08(bios, ramcfg.data + 0x02) & 0x04))
+                unk714 |= 0x00000010;
+        ram_wr32(fuc, 0x100714, unk714);
+
+        if (nv_ro08(bios, ramcfg.data + 0x02) & 0x01)
+                unk71c |= 0x00000100;
+        ram_wr32(fuc, 0x10071c, unk71c);
+
+        if (nv_ro08(bios, ramcfg.data + 0x02) & 0x02)
+                unk718 |= 0x00000100;
+        ram_wr32(fuc, 0x100718, unk718);
+
+        if (nv_ro08(bios, ramcfg.data + 0x02) & 0x10)
+                ram_wr32(fuc, 0x111100, 0x48000000); /*XXX*/
+
+        ram_mask(fuc, mr[0], 0x100, 0x100);
+        ram_nsec(fuc, 1000);
+        ram_mask(fuc, mr[0], 0x100, 0x000);
+        ram_nsec(fuc, 1000);
+
+        ram_nsec(fuc, 2000);
+        ram_nsec(fuc, 12000);
+
+        ram_wr32(fuc, 0x611200, 0x00003330);
+        if ( (nv_ro08(bios, rammap.data + 0x04) & 0x02))
+                ram_mask(fuc, 0x100200, 0x00000800, 0x00000800);
+        if ( (nv_ro08(bios, ramcfg.data + 0x02) & 0x10)) {
+                ram_mask(fuc, 0x111104, 0x00000180, 0x00000180);
+                ram_mask(fuc, 0x111100, 0x40000000, 0x00000000);
+        } else {
+                ram_mask(fuc, 0x111104, 0x00000600, 0x00000600);
+        }
+
+        if (mclk.pll) {
+                ram_mask(fuc, 0x004168, 0x00000001, 0x00000000);
+                ram_mask(fuc, 0x004168, 0x00000100, 0x00000000);
+        } else {
+                ram_mask(fuc, 0x004000, 0x00000001, 0x00000000);
+                ram_mask(fuc, 0x004128, 0x00000001, 0x00000000);
+                ram_mask(fuc, 0x004128, 0x00000100, 0x00000000);
+        }
+
+        return 0;
+}
+
+static int
+nva3_ram_prog(struct nouveau_fb *pfb)
+{
+        struct nouveau_device *device = nv_device(pfb);
+        struct nva3_ram *ram = (void *)pfb->ram;
+        struct nva3_ramfuc *fuc = &ram->fuc;
+        ram_exec(fuc, nouveau_boolopt(device->cfgopt, "NvMemExec", false));
+        return 0;
+}
+
+static void
+nva3_ram_tidy(struct nouveau_fb *pfb)
+{
+        struct nva3_ram *ram = (void *)pfb->ram;
+        struct nva3_ramfuc *fuc = &ram->fuc;
+        ram_exec(fuc, false);
+}
+
+static int
+nva3_ram_init(struct nouveau_object *object)
+{
+        struct nouveau_fb *pfb = (void *)object->parent;
+        struct nva3_ram   *ram = (void *)object;
+        int ret, i;
+
+        ret = nouveau_ram_init(&ram->base);
+        if (ret)
+                return ret;
+
+        /* prepare for ddr link training, and load training patterns */
+        switch (ram->base.type) {
+        case NV_MEM_TYPE_DDR3: {
+                static const u32 pattern[16] = {
+                        0xaaaaaaaa, 0xcccccccc, 0xdddddddd, 0xeeeeeeee,
+                        0x00000000, 0x11111111, 0x44444444, 0xdddddddd,
+                        0x33333333, 0x55555555, 0x77777777, 0x66666666,
+                        0x99999999, 0x88888888, 0xeeeeeeee, 0xbbbbbbbb,
+                };
+
+                nv_wr32(pfb, 0x100538, 0x10001ff6); /*XXX*/
+                nv_wr32(pfb, 0x1005a8, 0x0000ffff);
+                nv_mask(pfb, 0x10f800, 0x00000001, 0x00000001);
+                for (i = 0; i < 0x30; i++) {
+                        nv_wr32(pfb, 0x10f8c0, (i << 8) | i);
+                        nv_wr32(pfb, 0x10f8e0, (i << 8) | i);
+                        nv_wr32(pfb, 0x10f900, pattern[i % 16]);
+                        nv_wr32(pfb, 0x10f920, pattern[i % 16]);
+                }
+        }
+                break;
+        default:
+                break;
+        }
+
+        return 0;
+}
+
+static int
+nva3_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+              struct nouveau_oclass *oclass, void *data, u32 datasize,
+              struct nouveau_object **pobject)
+{
+        struct nva3_ram *ram;
+        int ret, i;
+
+        ret = nv50_ram_create(parent, engine, oclass, &ram);
+        *pobject = nv_object(ram);
+        if (ret)
+                return ret;
+
+        switch (ram->base.type) {
+        case NV_MEM_TYPE_DDR3:
+                ram->base.calc = nva3_ram_calc;
+                ram->base.prog = nva3_ram_prog;
+                ram->base.tidy = nva3_ram_tidy;
+                break;
+        default:
+                nv_warn(ram, "reclocking of this ram type unsupported\n");
+                return 0;
+        }
+
+        ram->fuc.r_0x004000 = ramfuc_reg(0x004000);
+        ram->fuc.r_0x004004 = ramfuc_reg(0x004004);
+        ram->fuc.r_0x004018 = ramfuc_reg(0x004018);
+        ram->fuc.r_0x004128 = ramfuc_reg(0x004128);
+        ram->fuc.r_0x004168 = ramfuc_reg(0x004168);
+        ram->fuc.r_0x100200 = ramfuc_reg(0x100200);
+        ram->fuc.r_0x100210 = ramfuc_reg(0x100210);
+        for (i = 0; i < 9; i++)
+                ram->fuc.r_0x100220[i] = ramfuc_reg(0x100220 + (i * 4));
+        ram->fuc.r_0x1002d0 = ramfuc_reg(0x1002d0);
+        ram->fuc.r_0x1002d4 = ramfuc_reg(0x1002d4);
+        ram->fuc.r_0x1002dc = ramfuc_reg(0x1002dc);
+        ram->fuc.r_0x10053c = ramfuc_reg(0x10053c);
+        ram->fuc.r_0x1005a0 = ramfuc_reg(0x1005a0);
+        ram->fuc.r_0x1005a4 = ramfuc_reg(0x1005a4);
+        ram->fuc.r_0x100714 = ramfuc_reg(0x100714);
+        ram->fuc.r_0x100718 = ramfuc_reg(0x100718);
+        ram->fuc.r_0x10071c = ramfuc_reg(0x10071c);
+        ram->fuc.r_0x100760 = ramfuc_reg(0x100760);
+        ram->fuc.r_0x1007a0 = ramfuc_reg(0x1007a0);
+        ram->fuc.r_0x1007e0 = ramfuc_reg(0x1007e0);
+        ram->fuc.r_0x10f804 = ramfuc_reg(0x10f804);
+        ram->fuc.r_0x1110e0 = ramfuc_reg(0x1110e0);
+        ram->fuc.r_0x111100 = ramfuc_reg(0x111100);
+        ram->fuc.r_0x111104 = ramfuc_reg(0x111104);
+        ram->fuc.r_0x611200 = ramfuc_reg(0x611200);
+
+        if (ram->base.ranks > 1) {
+                ram->fuc.r_mr[0] = ramfuc_reg2(0x1002c0, 0x1002c8);
+                ram->fuc.r_mr[1] = ramfuc_reg2(0x1002c4, 0x1002cc);
+                ram->fuc.r_mr[2] = ramfuc_reg2(0x1002e0, 0x1002e8);
+                ram->fuc.r_mr[3] = ramfuc_reg2(0x1002e4, 0x1002ec);
+        } else {
+                ram->fuc.r_mr[0] = ramfuc_reg(0x1002c0);
+                ram->fuc.r_mr[1] = ramfuc_reg(0x1002c4);
+                ram->fuc.r_mr[2] = ramfuc_reg(0x1002e0);
+                ram->fuc.r_mr[3] = ramfuc_reg(0x1002e4);
+        }
+
+        return 0;
+}
+
+struct nouveau_oclass
+nva3_ram_oclass = {
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nva3_ram_ctor,
+                .dtor = _nouveau_ram_dtor,
+                .init = nva3_ram_init,
+                .fini = _nouveau_ram_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvaa.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvaa.c
new file mode 100644
index 000000000000..00f2ca7e44a5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvaa.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv50.h"
+
+static int
+nvaa_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+              struct nouveau_oclass *oclass, void *data, u32 datasize,
+              struct nouveau_object **pobject)
+{
+        const u32 rsvd_head = ( 256 * 1024) >> 12; /* vga memory */
+        const u32 rsvd_tail = (1024 * 1024) >> 12; /* vbios etc */
+        struct nouveau_fb *pfb = nouveau_fb(parent);
+        struct nouveau_ram *ram;
+        int ret;
+
+        ret = nouveau_ram_create(parent, engine, oclass, &ram);
+        *pobject = nv_object(ram);
+        if (ret)
+                return ret;
+
+        ram->size = nv_rd32(pfb, 0x10020c);
+        ram->size = (ram->size & 0xffffff00) | ((ram->size & 0x000000ff) << 32);
+
+        ret = nouveau_mm_init(&pfb->vram, rsvd_head, (ram->size >> 12) -
+                              (rsvd_head + rsvd_tail), 1);
+        if (ret)
+                return ret;
+
+        ram->type   = NV_MEM_TYPE_STOLEN;
+        ram->stolen = (u64)nv_rd32(pfb, 0x100e10) << 12;
+        ram->get = nv50_ram_get;
+        ram->put = nv50_ram_put;
+        return 0;
+}
+
+struct nouveau_oclass
+nvaa_ram_oclass = {
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nvaa_ram_ctor,
+                .dtor = _nouveau_ram_dtor,
+                .init = _nouveau_ram_init,
+                .fini = _nouveau_ram_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvc0.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvc0.c
index cf97c4de4a6b..f464547c6bab 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvc0.c
@@ -23,9 +23,414 @@
  */
 
 #include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/pll.h>
+#include <subdev/bios/rammap.h>
+#include <subdev/bios/timing.h>
 #include <subdev/ltcg.h>
 
-#include "priv.h"
+#include <subdev/clock.h>
+#include <subdev/clock/pll.h>
+
+#include <core/option.h>
+
+#include "ramfuc.h"
+
+#include "nvc0.h"
+
+struct nvc0_ramfuc {
+        struct ramfuc base;
+
+        struct ramfuc_reg r_0x10fe20;
+        struct ramfuc_reg r_0x10fe24;
+        struct ramfuc_reg r_0x137320;
+        struct ramfuc_reg r_0x137330;
+
+        struct ramfuc_reg r_0x132000;
+        struct ramfuc_reg r_0x132004;
+        struct ramfuc_reg r_0x132100;
+
+        struct ramfuc_reg r_0x137390;
+
+        struct ramfuc_reg r_0x10f290;
+        struct ramfuc_reg r_0x10f294;
+        struct ramfuc_reg r_0x10f298;
+        struct ramfuc_reg r_0x10f29c;
+        struct ramfuc_reg r_0x10f2a0;
+
+        struct ramfuc_reg r_0x10f300;
+        struct ramfuc_reg r_0x10f338;
+        struct ramfuc_reg r_0x10f340;
+        struct ramfuc_reg r_0x10f344;
+        struct ramfuc_reg r_0x10f348;
+
+        struct ramfuc_reg r_0x10f910;
+        struct ramfuc_reg r_0x10f914;
+
+        struct ramfuc_reg r_0x100b0c;
+        struct ramfuc_reg r_0x10f050;
+        struct ramfuc_reg r_0x10f090;
+        struct ramfuc_reg r_0x10f200;
+        struct ramfuc_reg r_0x10f210;
+        struct ramfuc_reg r_0x10f310;
+        struct ramfuc_reg r_0x10f314;
+        struct ramfuc_reg r_0x10f610;
+        struct ramfuc_reg r_0x10f614;
+        struct ramfuc_reg r_0x10f800;
+        struct ramfuc_reg r_0x10f808;
+        struct ramfuc_reg r_0x10f824;
+        struct ramfuc_reg r_0x10f830;
+        struct ramfuc_reg r_0x10f988;
+        struct ramfuc_reg r_0x10f98c;
+        struct ramfuc_reg r_0x10f990;
+        struct ramfuc_reg r_0x10f998;
+        struct ramfuc_reg r_0x10f9b0;
+        struct ramfuc_reg r_0x10f9b4;
+        struct ramfuc_reg r_0x10fb04;
+        struct ramfuc_reg r_0x10fb08;
+        struct ramfuc_reg r_0x137300;
+        struct ramfuc_reg r_0x137310;
+        struct ramfuc_reg r_0x137360;
+        struct ramfuc_reg r_0x1373ec;
+        struct ramfuc_reg r_0x1373f0;
+        struct ramfuc_reg r_0x1373f8;
+
+        struct ramfuc_reg r_0x61c140;
+        struct ramfuc_reg r_0x611200;
+
+        struct ramfuc_reg r_0x13d8f4;
+};
+
+struct nvc0_ram {
+        struct nouveau_ram base;
+        struct nvc0_ramfuc fuc;
+        struct nvbios_pll refpll;
+        struct nvbios_pll mempll;
+};
+
+static void
+nvc0_ram_train(struct nvc0_ramfuc *fuc, u32 magic)
+{
+        struct nvc0_ram *ram = container_of(fuc, typeof(*ram), fuc);
+        struct nouveau_fb *pfb = nouveau_fb(ram);
+        u32 part = nv_rd32(pfb, 0x022438), i;
+        u32 mask = nv_rd32(pfb, 0x022554);
+        u32 addr = 0x110974;
+
+        ram_wr32(fuc, 0x10f910, magic);
+        ram_wr32(fuc, 0x10f914, magic);
+
+        for (i = 0; (magic & 0x80000000) && i < part; addr += 0x1000, i++) {
+                if (mask & (1 << i))
+                        continue;
+                ram_wait(fuc, addr, 0x0000000f, 0x00000000, 500000);
+        }
+}
+
+static int
+nvc0_ram_calc(struct nouveau_fb *pfb, u32 freq)
+{
+        struct nouveau_clock *clk = nouveau_clock(pfb);
+        struct nouveau_bios *bios = nouveau_bios(pfb);
+        struct nvc0_ram *ram = (void *)pfb->ram;
+        struct nvc0_ramfuc *fuc = &ram->fuc;
+        struct bit_entry M;
+        u8  ver, cnt, strap;
+        u32 data;
+        struct {
+                u32 data;
+                u8  size;
+        } rammap, ramcfg, timing;
+        int ref, div, out;
+        int from, mode;
+        int N1, M1, P;
+        int ret;
+
+        /* lookup memory config data relevant to the target frequency */
+        rammap.data = nvbios_rammap_match(bios, freq / 1000, &ver, &rammap.size,
+                                         &cnt, &ramcfg.size);
+        if (!rammap.data || ver != 0x10 || rammap.size < 0x0e) {
+                nv_error(pfb, "invalid/missing rammap entry\n");
+                return -EINVAL;
+        }
+
+        /* locate specific data set for the attached memory */
+        if (bit_entry(bios, 'M', &M) || M.version != 2 || M.length < 3) {
+                nv_error(pfb, "invalid/missing memory table\n");
+                return -EINVAL;
+        }
+
+        strap = (nv_rd32(pfb, 0x101000) & 0x0000003c) >> 2;
+        data = nv_ro16(bios, M.offset + 1);
+        if (data)
+                strap = nv_ro08(bios, data + strap);
+
+        if (strap >= cnt) {
+                nv_error(pfb, "invalid ramcfg strap\n");
+                return -EINVAL;
+        }
+
+        ramcfg.data = rammap.data + rammap.size + (strap * ramcfg.size);
+        if (!ramcfg.data || ver != 0x10 || ramcfg.size < 0x0e) {
+                nv_error(pfb, "invalid/missing ramcfg entry\n");
+                return -EINVAL;
+        }
+
+        /* lookup memory timings, if bios says they're present */
+        strap = nv_ro08(bios, ramcfg.data + 0x01);
+        if (strap != 0xff) {
+                timing.data = nvbios_timing_entry(bios, strap, &ver,
+                                                 &timing.size);
+                if (!timing.data || ver != 0x10 || timing.size < 0x19) {
+                        nv_error(pfb, "invalid/missing timing entry\n");
+                        return -EINVAL;
+                }
+        } else {
+                timing.data = 0;
+        }
+
+        ret = ram_init(fuc, pfb);
+        if (ret)
+                return ret;
+
+        /* determine current mclk configuration */
+        from = !!(ram_rd32(fuc, 0x1373f0) & 0x00000002); /*XXX: ok? */
+
+        /* determine target mclk configuration */
+        if (!(ram_rd32(fuc, 0x137300) & 0x00000100))
+                ref = clk->read(clk, nv_clk_src_sppll0);
+        else
+                ref = clk->read(clk, nv_clk_src_sppll1);
+        div = max(min((ref * 2) / freq, (u32)65), (u32)2) - 2;
+        out = (ref * 2) / (div + 2);
+        mode = freq != out;
+
+        ram_mask(fuc, 0x137360, 0x00000002, 0x00000000);
+
+        if ((ram_rd32(fuc, 0x132000) & 0x00000002) || 0 /*XXX*/) {
+                ram_nuke(fuc, 0x132000);
+                ram_mask(fuc, 0x132000, 0x00000002, 0x00000002);
+                ram_mask(fuc, 0x132000, 0x00000002, 0x00000000);
+        }
+
+        if (mode == 1) {
+                ram_nuke(fuc, 0x10fe20);
+                ram_mask(fuc, 0x10fe20, 0x00000002, 0x00000002);
+                ram_mask(fuc, 0x10fe20, 0x00000002, 0x00000000);
+        }
+
+// 0x00020034 // 0x0000000a
+        ram_wr32(fuc, 0x132100, 0x00000001);
+
+        if (mode == 1 && from == 0) {
+                /* calculate refpll */
+                ret = nva3_pll_calc(nv_subdev(pfb), &ram->refpll,
+                                    ram->mempll.refclk, &N1, NULL, &M1, &P);
+                if (ret <= 0) {
+                        nv_error(pfb, "unable to calc refpll\n");
+                        return ret ? ret : -ERANGE;
+                }
+
+                ram_wr32(fuc, 0x10fe20, 0x20010000);
+                ram_wr32(fuc, 0x137320, 0x00000003);
+                ram_wr32(fuc, 0x137330, 0x81200006);
+                ram_wr32(fuc, 0x10fe24, (P << 16) | (N1 << 8) | M1);
+                ram_wr32(fuc, 0x10fe20, 0x20010001);
+                ram_wait(fuc, 0x137390, 0x00020000, 0x00020000, 64000);
+
+                /* calculate mempll */
+                ret = nva3_pll_calc(nv_subdev(pfb), &ram->mempll, freq,
+                                   &N1, NULL, &M1, &P);
+                if (ret <= 0) {
+                        nv_error(pfb, "unable to calc refpll\n");
+                        return ret ? ret : -ERANGE;
+                }
+
+                ram_wr32(fuc, 0x10fe20, 0x20010005);
+                ram_wr32(fuc, 0x132004, (P << 16) | (N1 << 8) | M1);
+                ram_wr32(fuc, 0x132000, 0x18010101);
+                ram_wait(fuc, 0x137390, 0x00000002, 0x00000002, 64000);
+        } else
+        if (mode == 0) {
+                ram_wr32(fuc, 0x137300, 0x00000003);
+        }
+
+        if (from == 0) {
+                ram_nuke(fuc, 0x10fb04);
+                ram_mask(fuc, 0x10fb04, 0x0000ffff, 0x00000000);
+                ram_nuke(fuc, 0x10fb08);
+                ram_mask(fuc, 0x10fb08, 0x0000ffff, 0x00000000);
+                ram_wr32(fuc, 0x10f988, 0x2004ff00);
+                ram_wr32(fuc, 0x10f98c, 0x003fc040);
+                ram_wr32(fuc, 0x10f990, 0x20012001);
+                ram_wr32(fuc, 0x10f998, 0x00011a00);
+                ram_wr32(fuc, 0x13d8f4, 0x00000000);
+        } else {
+                ram_wr32(fuc, 0x10f988, 0x20010000);
+                ram_wr32(fuc, 0x10f98c, 0x00000000);
+                ram_wr32(fuc, 0x10f990, 0x20012001);
+                ram_wr32(fuc, 0x10f998, 0x00010a00);
+        }
+
+        if (from == 0) {
+// 0x00020039 // 0x000000ba
+        }
+
+// 0x0002003a // 0x00000002
+        ram_wr32(fuc, 0x100b0c, 0x00080012);
+// 0x00030014 // 0x00000000 // 0x02b5f070
+// 0x00030014 // 0x00010000 // 0x02b5f070
+        ram_wr32(fuc, 0x611200, 0x00003300);
+// 0x00020034 // 0x0000000a
+// 0x00030020 // 0x00000001 // 0x00000000
+
+        ram_mask(fuc, 0x10f200, 0x00000800, 0x00000000);
+        ram_wr32(fuc, 0x10f210, 0x00000000);
+        ram_nsec(fuc, 1000);
+        if (mode == 0)
+                nvc0_ram_train(fuc, 0x000c1001);
+        ram_wr32(fuc, 0x10f310, 0x00000001);
+        ram_nsec(fuc, 1000);
+        ram_wr32(fuc, 0x10f090, 0x00000061);
+        ram_wr32(fuc, 0x10f090, 0xc000007f);
+        ram_nsec(fuc, 1000);
+
+        if (from == 0) {
+                ram_wr32(fuc, 0x10f824, 0x00007fd4);
+        } else {
+                ram_wr32(fuc, 0x1373ec, 0x00020404);
+        }
+
+        if (mode == 0) {
+                ram_mask(fuc, 0x10f808, 0x00080000, 0x00000000);
+                ram_mask(fuc, 0x10f200, 0x00008000, 0x00008000);
+                ram_wr32(fuc, 0x10f830, 0x41500010);
+                ram_mask(fuc, 0x10f830, 0x01000000, 0x00000000);
+                ram_mask(fuc, 0x132100, 0x00000100, 0x00000100);
+                ram_wr32(fuc, 0x10f050, 0xff000090);
+                ram_wr32(fuc, 0x1373ec, 0x00020f0f);
+                ram_wr32(fuc, 0x1373f0, 0x00000003);
+                ram_wr32(fuc, 0x137310, 0x81201616);
+                ram_wr32(fuc, 0x132100, 0x00000001);
+// 0x00020039 // 0x000000ba
+                ram_wr32(fuc, 0x10f830, 0x00300017);
+                ram_wr32(fuc, 0x1373f0, 0x00000001);
+                ram_wr32(fuc, 0x10f824, 0x00007e77);
+                ram_wr32(fuc, 0x132000, 0x18030001);
+                ram_wr32(fuc, 0x10f090, 0x4000007e);
+                ram_nsec(fuc, 2000);
+                ram_wr32(fuc, 0x10f314, 0x00000001);
+                ram_wr32(fuc, 0x10f210, 0x80000000);
+                ram_wr32(fuc, 0x10f338, 0x00300220);
+                ram_wr32(fuc, 0x10f300, 0x0000011d);
+                ram_nsec(fuc, 1000);
+                ram_wr32(fuc, 0x10f290, 0x02060505);
+                ram_wr32(fuc, 0x10f294, 0x34208288);
+                ram_wr32(fuc, 0x10f298, 0x44050411);
+                ram_wr32(fuc, 0x10f29c, 0x0000114c);
+                ram_wr32(fuc, 0x10f2a0, 0x42e10069);
+                ram_wr32(fuc, 0x10f614, 0x40044f77);
+                ram_wr32(fuc, 0x10f610, 0x40044f77);
+                ram_wr32(fuc, 0x10f344, 0x00600009);
+                ram_nsec(fuc, 1000);
+                ram_wr32(fuc, 0x10f348, 0x00700008);
+                ram_wr32(fuc, 0x61c140, 0x19240000);
+                ram_wr32(fuc, 0x10f830, 0x00300017);
+                nvc0_ram_train(fuc, 0x80021001);
+                nvc0_ram_train(fuc, 0x80081001);
+                ram_wr32(fuc, 0x10f340, 0x00500004);
+                ram_nsec(fuc, 1000);
+                ram_wr32(fuc, 0x10f830, 0x01300017);
+                ram_wr32(fuc, 0x10f830, 0x00300017);
+// 0x00030020 // 0x00000000 // 0x00000000
+// 0x00020034 // 0x0000000b
+                ram_wr32(fuc, 0x100b0c, 0x00080028);
+                ram_wr32(fuc, 0x611200, 0x00003330);
+        } else {
+                ram_wr32(fuc, 0x10f800, 0x00001800);
+                ram_wr32(fuc, 0x13d8f4, 0x00000000);
+                ram_wr32(fuc, 0x1373ec, 0x00020404);
+                ram_wr32(fuc, 0x1373f0, 0x00000003);
+                ram_wr32(fuc, 0x10f830, 0x40700010);
+                ram_wr32(fuc, 0x10f830, 0x40500010);
+                ram_wr32(fuc, 0x13d8f4, 0x00000000);
+                ram_wr32(fuc, 0x1373f8, 0x00000000);
+                ram_wr32(fuc, 0x132100, 0x00000101);
+                ram_wr32(fuc, 0x137310, 0x89201616);
+                ram_wr32(fuc, 0x10f050, 0xff000090);
+                ram_wr32(fuc, 0x1373ec, 0x00030404);
+                ram_wr32(fuc, 0x1373f0, 0x00000002);
+        // 0x00020039 // 0x00000011
+                ram_wr32(fuc, 0x132100, 0x00000001);
+                ram_wr32(fuc, 0x1373f8, 0x00002000);
+                ram_nsec(fuc, 2000);
+                ram_wr32(fuc, 0x10f808, 0x7aaa0050);
+                ram_wr32(fuc, 0x10f830, 0x00500010);
+                ram_wr32(fuc, 0x10f200, 0x00ce1000);
+                ram_wr32(fuc, 0x10f090, 0x4000007e);
+                ram_nsec(fuc, 2000);
+                ram_wr32(fuc, 0x10f314, 0x00000001);
+                ram_wr32(fuc, 0x10f210, 0x80000000);
+                ram_wr32(fuc, 0x10f338, 0x00300200);
+                ram_wr32(fuc, 0x10f300, 0x0000084d);
+                ram_nsec(fuc, 1000);
+                ram_wr32(fuc, 0x10f290, 0x0b343825);
+                ram_wr32(fuc, 0x10f294, 0x3483028e);
+                ram_wr32(fuc, 0x10f298, 0x440c0600);
+                ram_wr32(fuc, 0x10f29c, 0x0000214c);
+                ram_wr32(fuc, 0x10f2a0, 0x42e20069);
+                ram_wr32(fuc, 0x10f200, 0x00ce0000);
+                ram_wr32(fuc, 0x10f614, 0x60044e77);
+                ram_wr32(fuc, 0x10f610, 0x60044e77);
+                ram_wr32(fuc, 0x10f340, 0x00500000);
+                ram_nsec(fuc, 1000);
+                ram_wr32(fuc, 0x10f344, 0x00600228);
+                ram_nsec(fuc, 1000);
+                ram_wr32(fuc, 0x10f348, 0x00700000);
+                ram_wr32(fuc, 0x13d8f4, 0x00000000);
+                ram_wr32(fuc, 0x61c140, 0x09a40000);
+
+                nvc0_ram_train(fuc, 0x800e1008);
+
+                ram_nsec(fuc, 1000);
+                ram_wr32(fuc, 0x10f800, 0x00001804);
+        // 0x00030020 // 0x00000000 // 0x00000000
+        // 0x00020034 // 0x0000000b
+                ram_wr32(fuc, 0x13d8f4, 0x00000000);
+                ram_wr32(fuc, 0x100b0c, 0x00080028);
+                ram_wr32(fuc, 0x611200, 0x00003330);
+                ram_nsec(fuc, 100000);
+                ram_wr32(fuc, 0x10f9b0, 0x05313f41);
+                ram_wr32(fuc, 0x10f9b4, 0x00002f50);
+
+                nvc0_ram_train(fuc, 0x010c1001);
+        }
+
+        ram_mask(fuc, 0x10f200, 0x00000800, 0x00000800);
+// 0x00020016 // 0x00000000
+
+        if (mode == 0)
+                ram_mask(fuc, 0x132000, 0x00000001, 0x00000000);
+        return 0;
+}
+
+static int
+nvc0_ram_prog(struct nouveau_fb *pfb)
+{
+        struct nouveau_device *device = nv_device(pfb);
+        struct nvc0_ram *ram = (void *)pfb->ram;
+        struct nvc0_ramfuc *fuc = &ram->fuc;
+        ram_exec(fuc, nouveau_boolopt(device->cfgopt, "NvMemExec", false));
+        return 0;
+}
+
+static void
+nvc0_ram_tidy(struct nouveau_fb *pfb)
+{
+        struct nvc0_ram *ram = (void *)pfb->ram;
+        struct nvc0_ramfuc *fuc = &ram->fuc;
+        ram_exec(fuc, false);
+}
 
 extern const u8 nvc0_pte_storage_type_map[256];
 
@@ -110,10 +515,9 @@ nvc0_ram_get(struct nouveau_fb *pfb, u64 size, u32 align, u32 ncmin,
         return 0;
 }
 
-static int
-nvc0_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
-                struct nouveau_oclass *oclass, void *data, u32 size,
-                struct nouveau_object **pobject)
+int
+nvc0_ram_create_(struct nouveau_object *parent, struct nouveau_object *engine,
+                 struct nouveau_oclass *oclass, int size, void **pobject)
 {
         struct nouveau_fb *pfb = nouveau_fb(parent);
         struct nouveau_bios *bios = nouveau_bios(pfb);
@@ -127,8 +531,8 @@ nvc0_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
         bool uniform = true;
         int ret, part;
 
-        ret = nouveau_ram_create(parent, engine, oclass, &ram);
-        *pobject = nv_object(ram);
+        ret = nouveau_ram_create_(parent, engine, oclass, size, pobject);
+        ram = *pobject;
         if (ret)
                 return ret;
 
@@ -182,13 +586,158 @@ nvc0_ram_create(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
+static int
+nvc0_ram_init(struct nouveau_object *object)
+{
+        struct nouveau_fb *pfb = (void *)object->parent;
+        struct nvc0_ram   *ram = (void *)object;
+        int ret, i;
+
+        ret = nouveau_ram_init(&ram->base);
+        if (ret)
+                return ret;
+
+        /* prepare for ddr link training, and load training patterns */
+        switch (ram->base.type) {
+        case NV_MEM_TYPE_GDDR5: {
+                static const u8  train0[] = {
+                        0x00, 0xff, 0x55, 0xaa, 0x33, 0xcc,
+                        0x00, 0xff, 0xff, 0x00, 0xff, 0x00,
+                };
+                static const u32 train1[] = {
+                        0x00000000, 0xffffffff,
+                        0x55555555, 0xaaaaaaaa,
+                        0x33333333, 0xcccccccc,
+                        0xf0f0f0f0, 0x0f0f0f0f,
+                        0x00ff00ff, 0xff00ff00,
+                        0x0000ffff, 0xffff0000,
+                };
+
+                for (i = 0; i < 0x30; i++) {
+                        nv_wr32(pfb, 0x10f968, 0x00000000 | (i << 8));
+                        nv_wr32(pfb, 0x10f96c, 0x00000000 | (i << 8));
+                        nv_wr32(pfb, 0x10f920, 0x00000100 | train0[i % 12]);
+                        nv_wr32(pfb, 0x10f924, 0x00000100 | train0[i % 12]);
+                        nv_wr32(pfb, 0x10f918,              train1[i % 12]);
+                        nv_wr32(pfb, 0x10f91c,              train1[i % 12]);
+                        nv_wr32(pfb, 0x10f920, 0x00000000 | train0[i % 12]);
+                        nv_wr32(pfb, 0x10f924, 0x00000000 | train0[i % 12]);
+                        nv_wr32(pfb, 0x10f918,              train1[i % 12]);
+                        nv_wr32(pfb, 0x10f91c,              train1[i % 12]);
+                }
+        }       break;
+        default:
+                break;
+        }
+
+        return 0;
+}
+
+static int
+nvc0_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+              struct nouveau_oclass *oclass, void *data, u32 size,
+              struct nouveau_object **pobject)
+{
+        struct nouveau_bios *bios = nouveau_bios(parent);
+        struct nvc0_ram *ram;
+        int ret;
+
+        ret = nvc0_ram_create(parent, engine, oclass, &ram);
+        *pobject = nv_object(ram);
+        if (ret)
+                return ret;
+
+        ret = nvbios_pll_parse(bios, 0x0c, &ram->refpll);
+        if (ret) {
+                nv_error(ram, "mclk refpll data not found\n");
+                return ret;
+        }
+
+        ret = nvbios_pll_parse(bios, 0x04, &ram->mempll);
+        if (ret) {
+                nv_error(ram, "mclk pll data not found\n");
+                return ret;
+        }
+
+        switch (ram->base.type) {
+        case NV_MEM_TYPE_GDDR5:
+                ram->base.calc = nvc0_ram_calc;
+                ram->base.prog = nvc0_ram_prog;
+                ram->base.tidy = nvc0_ram_tidy;
+                break;
+        default:
+                nv_warn(ram, "reclocking of this ram type unsupported\n");
+                return 0;
+        }
+
+        ram->fuc.r_0x10fe20 = ramfuc_reg(0x10fe20);
+        ram->fuc.r_0x10fe24 = ramfuc_reg(0x10fe24);
+        ram->fuc.r_0x137320 = ramfuc_reg(0x137320);
+        ram->fuc.r_0x137330 = ramfuc_reg(0x137330);
+
+        ram->fuc.r_0x132000 = ramfuc_reg(0x132000);
+        ram->fuc.r_0x132004 = ramfuc_reg(0x132004);
+        ram->fuc.r_0x132100 = ramfuc_reg(0x132100);
+
+        ram->fuc.r_0x137390 = ramfuc_reg(0x137390);
+
+        ram->fuc.r_0x10f290 = ramfuc_reg(0x10f290);
+        ram->fuc.r_0x10f294 = ramfuc_reg(0x10f294);
+        ram->fuc.r_0x10f298 = ramfuc_reg(0x10f298);
+        ram->fuc.r_0x10f29c = ramfuc_reg(0x10f29c);
+        ram->fuc.r_0x10f2a0 = ramfuc_reg(0x10f2a0);
+
+        ram->fuc.r_0x10f300 = ramfuc_reg(0x10f300);
+        ram->fuc.r_0x10f338 = ramfuc_reg(0x10f338);
+        ram->fuc.r_0x10f340 = ramfuc_reg(0x10f340);
+        ram->fuc.r_0x10f344 = ramfuc_reg(0x10f344);
+        ram->fuc.r_0x10f348 = ramfuc_reg(0x10f348);
+
+        ram->fuc.r_0x10f910 = ramfuc_reg(0x10f910);
+        ram->fuc.r_0x10f914 = ramfuc_reg(0x10f914);
+
+        ram->fuc.r_0x100b0c = ramfuc_reg(0x100b0c);
+        ram->fuc.r_0x10f050 = ramfuc_reg(0x10f050);
+        ram->fuc.r_0x10f090 = ramfuc_reg(0x10f090);
+        ram->fuc.r_0x10f200 = ramfuc_reg(0x10f200);
+        ram->fuc.r_0x10f210 = ramfuc_reg(0x10f210);
+        ram->fuc.r_0x10f310 = ramfuc_reg(0x10f310);
+        ram->fuc.r_0x10f314 = ramfuc_reg(0x10f314);
+        ram->fuc.r_0x10f610 = ramfuc_reg(0x10f610);
+        ram->fuc.r_0x10f614 = ramfuc_reg(0x10f614);
+        ram->fuc.r_0x10f800 = ramfuc_reg(0x10f800);
+        ram->fuc.r_0x10f808 = ramfuc_reg(0x10f808);
+        ram->fuc.r_0x10f824 = ramfuc_reg(0x10f824);
+        ram->fuc.r_0x10f830 = ramfuc_reg(0x10f830);
+        ram->fuc.r_0x10f988 = ramfuc_reg(0x10f988);
+        ram->fuc.r_0x10f98c = ramfuc_reg(0x10f98c);
+        ram->fuc.r_0x10f990 = ramfuc_reg(0x10f990);
+        ram->fuc.r_0x10f998 = ramfuc_reg(0x10f998);
+        ram->fuc.r_0x10f9b0 = ramfuc_reg(0x10f9b0);
+        ram->fuc.r_0x10f9b4 = ramfuc_reg(0x10f9b4);
+        ram->fuc.r_0x10fb04 = ramfuc_reg(0x10fb04);
+        ram->fuc.r_0x10fb08 = ramfuc_reg(0x10fb08);
+        ram->fuc.r_0x137310 = ramfuc_reg(0x137300);
+        ram->fuc.r_0x137310 = ramfuc_reg(0x137310);
+        ram->fuc.r_0x137360 = ramfuc_reg(0x137360);
+        ram->fuc.r_0x1373ec = ramfuc_reg(0x1373ec);
+        ram->fuc.r_0x1373f0 = ramfuc_reg(0x1373f0);
+        ram->fuc.r_0x1373f8 = ramfuc_reg(0x1373f8);
+
+        ram->fuc.r_0x61c140 = ramfuc_reg(0x61c140);
+        ram->fuc.r_0x611200 = ramfuc_reg(0x611200);
+
+        ram->fuc.r_0x13d8f4 = ramfuc_reg(0x13d8f4);
+        return 0;
+}
+
 struct nouveau_oclass
 nvc0_ram_oclass = {
         .handle = 0,
         .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nvc0_ram_create,
+                .ctor = nvc0_ram_ctor,
                 .dtor = _nouveau_ram_dtor,
-                .init = _nouveau_ram_init,
+                .init = nvc0_ram_init,
                 .fini = _nouveau_ram_fini,
         }
 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnve0.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnve0.c
new file mode 100644
index 000000000000..bc86cfd084f6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnve0.c
@@ -0,0 +1,1264 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/gpio.h>
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/pll.h>
+#include <subdev/bios/init.h>
+#include <subdev/bios/rammap.h>
+#include <subdev/bios/timing.h>
+
+#include <subdev/clock.h>
+#include <subdev/clock/pll.h>
+
+#include <subdev/timer.h>
+
+#include <core/option.h>
+
+#include "nvc0.h"
+
+#include "ramfuc.h"
+
+struct nve0_ramfuc {
+        struct ramfuc base;
+
+        struct nvbios_pll refpll;
+        struct nvbios_pll mempll;
+
+        struct ramfuc_reg r_gpioMV;
+        u32 r_funcMV[2];
+        struct ramfuc_reg r_gpio2E;
+        u32 r_func2E[2];
+        struct ramfuc_reg r_gpiotrig;
+
+        struct ramfuc_reg r_0x132020;
+        struct ramfuc_reg r_0x132028;
+        struct ramfuc_reg r_0x132024;
+        struct ramfuc_reg r_0x132030;
+        struct ramfuc_reg r_0x132034;
+        struct ramfuc_reg r_0x132000;
+        struct ramfuc_reg r_0x132004;
+        struct ramfuc_reg r_0x132040;
+
+        struct ramfuc_reg r_0x10f248;
+        struct ramfuc_reg r_0x10f290;
+        struct ramfuc_reg r_0x10f294;
+        struct ramfuc_reg r_0x10f298;
+        struct ramfuc_reg r_0x10f29c;
+        struct ramfuc_reg r_0x10f2a0;
+        struct ramfuc_reg r_0x10f2a4;
+        struct ramfuc_reg r_0x10f2a8;
+        struct ramfuc_reg r_0x10f2ac;
+        struct ramfuc_reg r_0x10f2cc;
+        struct ramfuc_reg r_0x10f2e8;
+        struct ramfuc_reg r_0x10f250;
+        struct ramfuc_reg r_0x10f24c;
+        struct ramfuc_reg r_0x10fec4;
+        struct ramfuc_reg r_0x10fec8;
+        struct ramfuc_reg r_0x10f604;
+        struct ramfuc_reg r_0x10f614;
+        struct ramfuc_reg r_0x10f610;
+        struct ramfuc_reg r_0x100770;
+        struct ramfuc_reg r_0x100778;
+        struct ramfuc_reg r_0x10f224;
+
+        struct ramfuc_reg r_0x10f870;
+        struct ramfuc_reg r_0x10f698;
+        struct ramfuc_reg r_0x10f694;
+        struct ramfuc_reg r_0x10f6b8;
+        struct ramfuc_reg r_0x10f808;
+        struct ramfuc_reg r_0x10f670;
+        struct ramfuc_reg r_0x10f60c;
+        struct ramfuc_reg r_0x10f830;
+        struct ramfuc_reg r_0x1373ec;
+        struct ramfuc_reg r_0x10f800;
+        struct ramfuc_reg r_0x10f82c;
+
+        struct ramfuc_reg r_0x10f978;
+        struct ramfuc_reg r_0x10f910;
+        struct ramfuc_reg r_0x10f914;
+
+        struct ramfuc_reg r_mr[16]; /* MR0 - MR8, MR15 */
+
+        struct ramfuc_reg r_0x62c000;
+        struct ramfuc_reg r_0x10f200;
+        struct ramfuc_reg r_0x10f210;
+        struct ramfuc_reg r_0x10f310;
+        struct ramfuc_reg r_0x10f314;
+        struct ramfuc_reg r_0x10f318;
+        struct ramfuc_reg r_0x10f090;
+        struct ramfuc_reg r_0x10f69c;
+        struct ramfuc_reg r_0x10f824;
+        struct ramfuc_reg r_0x1373f0;
+        struct ramfuc_reg r_0x1373f4;
+        struct ramfuc_reg r_0x137320;
+        struct ramfuc_reg r_0x10f65c;
+        struct ramfuc_reg r_0x10f6bc;
+        struct ramfuc_reg r_0x100710;
+        struct ramfuc_reg r_0x10f750;
+};
+
+struct nve0_ram {
+        struct nouveau_ram base;
+        struct nve0_ramfuc fuc;
+        int from;
+        int mode;
+        int N1, fN1, M1, P1;
+        int N2, M2, P2;
+};
+
+/*******************************************************************************
+ * GDDR5
+ ******************************************************************************/
+static void
+train(struct nve0_ramfuc *fuc, u32 magic)
+{
+        struct nve0_ram *ram = container_of(fuc, typeof(*ram), fuc);
+        struct nouveau_fb *pfb = nouveau_fb(ram);
+        const int mc = nv_rd32(pfb, 0x02243c);
+        int i;
+
+        ram_mask(fuc, 0x10f910, 0xbc0e0000, magic);
+        ram_mask(fuc, 0x10f914, 0xbc0e0000, magic);
+        for (i = 0; i < mc; i++) {
+                const u32 addr = 0x110974 + (i * 0x1000);
+                ram_wait(fuc, addr, 0x0000000f, 0x00000000, 500000);
+        }
+}
+
+static void
+r1373f4_init(struct nve0_ramfuc *fuc)
+{
+        struct nve0_ram *ram = container_of(fuc, typeof(*ram), fuc);
+        const u32 mcoef = ((--ram->P2 << 28) | (ram->N2 << 8) | ram->M2);
+        const u32 rcoef = ((  ram->P1 << 16) | (ram->N1 << 8) | ram->M1);
+        const u32 runk0 = ram->fN1 << 16;
+        const u32 runk1 = ram->fN1;
+
+        if (ram->from == 2) {
+                ram_mask(fuc, 0x1373f4, 0x00000000, 0x00001100);
+                ram_mask(fuc, 0x1373f4, 0x00000000, 0x00000010);
+        } else {
+                ram_mask(fuc, 0x1373f4, 0x00000000, 0x00010010);
+        }
+
+        ram_mask(fuc, 0x1373f4, 0x00000003, 0x00000000);
+        ram_mask(fuc, 0x1373f4, 0x00000010, 0x00000000);
+
+        /* (re)program refpll, if required */
+        if ((ram_rd32(fuc, 0x132024) & 0xffffffff) != rcoef ||
+            (ram_rd32(fuc, 0x132034) & 0x0000ffff) != runk1) {
+                ram_mask(fuc, 0x132000, 0x00000001, 0x00000000);
+                ram_mask(fuc, 0x132020, 0x00000001, 0x00000000);
+                ram_wr32(fuc, 0x137320, 0x00000000);
+                ram_mask(fuc, 0x132030, 0xffff0000, runk0);
+                ram_mask(fuc, 0x132034, 0x0000ffff, runk1);
+                ram_wr32(fuc, 0x132024, rcoef);
+                ram_mask(fuc, 0x132028, 0x00080000, 0x00080000);
+                ram_mask(fuc, 0x132020, 0x00000001, 0x00000001);
+                ram_wait(fuc, 0x137390, 0x00020000, 0x00020000, 64000);
+                ram_mask(fuc, 0x132028, 0x00080000, 0x00000000);
+        }
+
+        /* (re)program mempll, if required */
+        if (ram->mode == 2) {
+                ram_mask(fuc, 0x1373f4, 0x00010000, 0x00000000);
+                ram_mask(fuc, 0x132000, 0x00000001, 0x00000000);
+                ram_mask(fuc, 0x132004, 0x103fffff, mcoef);
+                ram_mask(fuc, 0x132000, 0x00000001, 0x00000001);
+                ram_wait(fuc, 0x137390, 0x00000002, 0x00000002, 64000);
+                ram_mask(fuc, 0x1373f4, 0x00000000, 0x00001100);
+        } else {
+                ram_mask(fuc, 0x1373f4, 0x00000000, 0x00010100);
+        }
+
+        ram_mask(fuc, 0x1373f4, 0x00000000, 0x00000010);
+}
+
+static void
+r1373f4_fini(struct nve0_ramfuc *fuc, u32 ramcfg)
+{
+        struct nve0_ram *ram = container_of(fuc, typeof(*ram), fuc);
+        struct nouveau_bios *bios = nouveau_bios(ram);
+        u8 v0 = (nv_ro08(bios, ramcfg + 0x03) & 0xc0) >> 6;
+        u8 v1 = (nv_ro08(bios, ramcfg + 0x03) & 0x30) >> 4;
+        u32 tmp;
+
+        tmp = ram_rd32(fuc, 0x1373ec) & ~0x00030000;
+        ram_wr32(fuc, 0x1373ec, tmp | (v1 << 16));
+        ram_mask(fuc, 0x1373f0, (~ram->mode & 3), 0x00000000);
+        if (ram->mode == 2) {
+                ram_mask(fuc, 0x1373f4, 0x00000003, 0x000000002);
+                ram_mask(fuc, 0x1373f4, 0x00001100, 0x000000000);
+        } else {
+                ram_mask(fuc, 0x1373f4, 0x00000003, 0x000000001);
+                ram_mask(fuc, 0x1373f4, 0x00010000, 0x000000000);
+        }
+        ram_mask(fuc, 0x10f800, 0x00000030, (v0 ^ v1) << 4);
+}
+
+static int
+nve0_ram_calc_gddr5(struct nouveau_fb *pfb, u32 freq)
+{
+        struct nouveau_bios *bios = nouveau_bios(pfb);
+        struct nve0_ram *ram = (void *)pfb->ram;
+        struct nve0_ramfuc *fuc = &ram->fuc;
+        const u32 rammap = ram->base.rammap.data;
+        const u32 ramcfg = ram->base.ramcfg.data;
+        const u32 timing = ram->base.timing.data;
+        int vc = !(nv_ro08(bios, ramcfg + 0x02) & 0x08);
+        int mv = 1; /*XXX*/
+        u32 mask, data;
+
+        ram_mask(fuc, 0x10f808, 0x40000000, 0x40000000);
+        ram_wr32(fuc, 0x62c000, 0x0f0f0000);
+
+        /* MR1: turn termination on early, for some reason.. */
+        if ((ram->base.mr[1] & 0x03c) != 0x030)
+                ram_mask(fuc, mr[1], 0x03c, ram->base.mr[1] & 0x03c);
+
+        if (vc == 1 && ram_have(fuc, gpio2E)) {
+                u32 temp  = ram_mask(fuc, gpio2E, 0x3000, fuc->r_func2E[1]);
+                if (temp != ram_rd32(fuc, gpio2E)) {
+                        ram_wr32(fuc, gpiotrig, 1);
+                        ram_nsec(fuc, 20000);
+                }
+        }
+
+        ram_mask(fuc, 0x10f200, 0x00000800, 0x00000000);
+
+        ram_mask(fuc, 0x10f914, 0x01020000, 0x000c0000);
+        ram_mask(fuc, 0x10f910, 0x01020000, 0x000c0000);
+
+        ram_wr32(fuc, 0x10f210, 0x00000000); /* REFRESH_AUTO = 0 */
+        ram_nsec(fuc, 1000);
+        ram_wr32(fuc, 0x10f310, 0x00000001); /* REFRESH */
+        ram_nsec(fuc, 1000);
+
+        ram_mask(fuc, 0x10f200, 0x80000000, 0x80000000);
+        ram_wr32(fuc, 0x10f314, 0x00000001); /* PRECHARGE */
+        ram_mask(fuc, 0x10f200, 0x80000000, 0x00000000);
+        ram_wr32(fuc, 0x10f090, 0x00000061);
+        ram_wr32(fuc, 0x10f090, 0xc000007f);
+        ram_nsec(fuc, 1000);
+
+        ram_wr32(fuc, 0x10f698, 0x00000000);
+        ram_wr32(fuc, 0x10f69c, 0x00000000);
+
+        /*XXX: there does appear to be some kind of condition here, simply
+         *     modifying these bits in the vbios from the default pl0
+         *     entries shows no change.  however, the data does appear to
+         *     be correct and may be required for the transition back
+         */
+        mask = 0x800f07e0;
+        data = 0x00030000;
+        if (ram_rd32(fuc, 0x10f978) & 0x00800000)
+                data |= 0x00040000;
+
+        if (1) {
+                data |= 0x800807e0;
+                switch (nv_ro08(bios, ramcfg + 0x03) & 0xc0) {
+                case 0xc0: data &= ~0x00000040; break;
+                case 0x80: data &= ~0x00000100; break;
+                case 0x40: data &= ~0x80000000; break;
+                case 0x00: data &= ~0x00000400; break;
+                }
+
+                switch (nv_ro08(bios, ramcfg + 0x03) & 0x30) {
+                case 0x30: data &= ~0x00000020; break;
+                case 0x20: data &= ~0x00000080; break;
+                case 0x10: data &= ~0x00080000; break;
+                case 0x00: data &= ~0x00000200; break;
+                }
+        }
+
+        if (nv_ro08(bios, ramcfg + 0x02) & 0x80)
+                mask |= 0x03000000;
+        if (nv_ro08(bios, ramcfg + 0x02) & 0x40)
+                mask |= 0x00002000;
+        if (nv_ro08(bios, ramcfg + 0x07) & 0x10)
+                mask |= 0x00004000;
+        if (nv_ro08(bios, ramcfg + 0x07) & 0x08)
+                mask |= 0x00000003;
+        else {
+                mask |= 0x34000000;
+                if (ram_rd32(fuc, 0x10f978) & 0x00800000)
+                        mask |= 0x40000000;
+        }
+        ram_mask(fuc, 0x10f824, mask, data);
+
+        ram_mask(fuc, 0x132040, 0x00010000, 0x00000000);
+
+        if (ram->from == 2 && ram->mode != 2) {
+                ram_mask(fuc, 0x10f808, 0x00080000, 0x00000000);
+                ram_mask(fuc, 0x10f200, 0x00008000, 0x00008000);
+                ram_mask(fuc, 0x10f800, 0x00000000, 0x00000004);
+                ram_mask(fuc, 0x10f830, 0x00008000, 0x01040010);
+                ram_mask(fuc, 0x10f830, 0x01000000, 0x00000000);
+                r1373f4_init(fuc);
+                ram_mask(fuc, 0x1373f0, 0x00000002, 0x00000001);
+                r1373f4_fini(fuc, ramcfg);
+                ram_mask(fuc, 0x10f830, 0x00c00000, 0x00240001);
+        } else
+        if (ram->from != 2 && ram->mode != 2) {
+                r1373f4_init(fuc);
+                r1373f4_fini(fuc, ramcfg);
+        }
+
+        if (ram_have(fuc, gpioMV)) {
+                u32 temp  = ram_mask(fuc, gpioMV, 0x3000, fuc->r_funcMV[mv]);
+                if (temp != ram_rd32(fuc, gpioMV)) {
+                        ram_wr32(fuc, gpiotrig, 1);
+                        ram_nsec(fuc, 64000);
+                }
+        }
+
+        if ( (nv_ro08(bios, ramcfg + 0x02) & 0x40) ||
+             (nv_ro08(bios, ramcfg + 0x07) & 0x10)) {
+                ram_mask(fuc, 0x132040, 0x00010000, 0x00010000);
+                ram_nsec(fuc, 20000);
+        }
+
+        if (ram->from != 2 && ram->mode == 2) {
+                ram_mask(fuc, 0x10f800, 0x00000004, 0x00000000);
+                ram_mask(fuc, 0x1373f0, 0x00000000, 0x00000002);
+                ram_mask(fuc, 0x10f830, 0x00800001, 0x00408010);
+                r1373f4_init(fuc);
+                r1373f4_fini(fuc, ramcfg);
+                ram_mask(fuc, 0x10f808, 0x00000000, 0x00080000);
+                ram_mask(fuc, 0x10f200, 0x00808000, 0x00800000);
+        } else
+        if (ram->from == 2 && ram->mode == 2) {
+                ram_mask(fuc, 0x10f800, 0x00000004, 0x00000000);
+                r1373f4_init(fuc);
+                r1373f4_fini(fuc, ramcfg);
+        }
+
+        if (ram->mode != 2) /*XXX*/ {
+                if (nv_ro08(bios, ramcfg + 0x07) & 0x40)
+                        ram_mask(fuc, 0x10f670, 0x80000000, 0x80000000);
+        }
+
+        data = (nv_ro08(bios, rammap + 0x11) & 0x0c) >> 2;
+        ram_wr32(fuc, 0x10f65c, 0x00000011 * data);
+        ram_wr32(fuc, 0x10f6b8, 0x01010101 * nv_ro08(bios, ramcfg + 0x09));
+        ram_wr32(fuc, 0x10f6bc, 0x01010101 * nv_ro08(bios, ramcfg + 0x09));
+
+        data = nv_ro08(bios, ramcfg + 0x04);
+        if (!(nv_ro08(bios, ramcfg + 0x07) & 0x08)) {
+                ram_wr32(fuc, 0x10f698, 0x01010101 * data);
+                ram_wr32(fuc, 0x10f69c, 0x01010101 * data);
+        }
+
+        if (ram->mode != 2) {
+                u32 temp = ram_rd32(fuc, 0x10f694) & ~0xff00ff00;
+                ram_wr32(fuc, 0x10f694, temp | (0x01000100 * data));
+        }
+
+        if (ram->mode == 2 && (nv_ro08(bios, ramcfg + 0x08) & 0x10))
+                data = 0x00000080;
+        else
+                data = 0x00000000;
+        ram_mask(fuc, 0x10f60c, 0x00000080, data);
+
+        mask = 0x00070000;
+        data = 0x00000000;
+        if (!(nv_ro08(bios, ramcfg + 0x02) & 0x80))
+                data |= 0x03000000;
+        if (!(nv_ro08(bios, ramcfg + 0x02) & 0x40))
+                data |= 0x00002000;
+        if (!(nv_ro08(bios, ramcfg + 0x07) & 0x10))
+                data |= 0x00004000;
+        if (!(nv_ro08(bios, ramcfg + 0x07) & 0x08))
+                data |= 0x00000003;
+        else
+                data |= 0x74000000;
+        ram_mask(fuc, 0x10f824, mask, data);
+
+        if (nv_ro08(bios, ramcfg + 0x01) & 0x08)
+                data = 0x00000000;
+        else
+                data = 0x00001000;
+        ram_mask(fuc, 0x10f200, 0x00001000, data);
+
+        if (ram_rd32(fuc, 0x10f670) & 0x80000000) {
+                ram_nsec(fuc, 10000);
+                ram_mask(fuc, 0x10f670, 0x80000000, 0x00000000);
+        }
+
+        if (nv_ro08(bios, ramcfg + 0x08) & 0x01)
+                data = 0x00100000;
+        else
+                data = 0x00000000;
+        ram_mask(fuc, 0x10f82c, 0x00100000, data);
+
+        data = 0x00000000;
+        if (nv_ro08(bios, ramcfg + 0x08) & 0x08)
+                data |= 0x00002000;
+        if (nv_ro08(bios, ramcfg + 0x08) & 0x04)
+                data |= 0x00001000;
+        if (nv_ro08(bios, ramcfg + 0x08) & 0x02)
+                data |= 0x00004000;
+        ram_mask(fuc, 0x10f830, 0x00007000, data);
+
+        /* PFB timing */
+        ram_mask(fuc, 0x10f248, 0xffffffff, nv_ro32(bios, timing + 0x28));
+        ram_mask(fuc, 0x10f290, 0xffffffff, nv_ro32(bios, timing + 0x00));
+        ram_mask(fuc, 0x10f294, 0xffffffff, nv_ro32(bios, timing + 0x04));
+        ram_mask(fuc, 0x10f298, 0xffffffff, nv_ro32(bios, timing + 0x08));
+        ram_mask(fuc, 0x10f29c, 0xffffffff, nv_ro32(bios, timing + 0x0c));
+        ram_mask(fuc, 0x10f2a0, 0xffffffff, nv_ro32(bios, timing + 0x10));
+        ram_mask(fuc, 0x10f2a4, 0xffffffff, nv_ro32(bios, timing + 0x14));
+        ram_mask(fuc, 0x10f2a8, 0xffffffff, nv_ro32(bios, timing + 0x18));
+        ram_mask(fuc, 0x10f2ac, 0xffffffff, nv_ro32(bios, timing + 0x1c));
+        ram_mask(fuc, 0x10f2cc, 0xffffffff, nv_ro32(bios, timing + 0x20));
+        ram_mask(fuc, 0x10f2e8, 0xffffffff, nv_ro32(bios, timing + 0x24));
+
+        data = (nv_ro08(bios, ramcfg + 0x02) & 0x03) << 8;
+        if (nv_ro08(bios, ramcfg + 0x01) & 0x10)
+                data |= 0x70000000;
+        ram_mask(fuc, 0x10f604, 0x70000300, data);
+
+        data = (nv_ro08(bios, timing + 0x30) & 0x07) << 28;
+        if (nv_ro08(bios, ramcfg + 0x01) & 0x01)
+                data |= 0x00000100;
+        ram_mask(fuc, 0x10f614, 0x70000000, data);
+
+        data = (nv_ro08(bios, timing + 0x30) & 0x07) << 28;
+        if (nv_ro08(bios, ramcfg + 0x01) & 0x02)
+                data |= 0x00000100;
+        ram_mask(fuc, 0x10f610, 0x70000000, data);
+
+        mask = 0x33f00000;
+        data = 0x00000000;
+        if (!(nv_ro08(bios, ramcfg + 0x01) & 0x04))
+                data |= 0x20200000;
+        if (!(nv_ro08(bios, ramcfg + 0x07) & 0x80))
+                data |= 0x12800000;
+        /*XXX: see note above about there probably being some condition
+         *     for the 10f824 stuff that uses ramcfg 3...
+         */
+        if ( (nv_ro08(bios, ramcfg + 0x03) & 0xf0)) {
+                if (nv_ro08(bios, rammap + 0x08) & 0x0c) {
+                        if (!(nv_ro08(bios, ramcfg + 0x07) & 0x80))
+                                mask |= 0x00000020;
+                        else
+                                data |= 0x00000020;
+                        mask |= 0x00000004;
+                }
+        } else {
+                mask |= 0x40000020;
+                data |= 0x00000004;
+        }
+
+        ram_mask(fuc, 0x10f808, mask, data);
+
+        data = nv_ro08(bios, ramcfg + 0x03) & 0x0f;
+        ram_wr32(fuc, 0x10f870, 0x11111111 * data);
+
+        data = nv_ro08(bios, ramcfg + 0x02) & 0x03;
+        if (nv_ro08(bios, ramcfg + 0x01) & 0x10)
+                data |= 0x00000004;
+        if ((nv_rd32(bios, 0x100770) & 0x00000004) != (data & 0x00000004)) {
+                ram_wr32(fuc, 0x10f750, 0x04000009);
+                ram_wr32(fuc, 0x100710, 0x00000000);
+                ram_wait(fuc, 0x100710, 0x80000000, 0x80000000, 200000);
+        }
+        ram_mask(fuc, 0x100770, 0x00000007, data);
+
+        data = (nv_ro08(bios, timing + 0x30) & 0x07) << 8;
+        if (nv_ro08(bios, ramcfg + 0x01) & 0x01)
+                data |= 0x80000000;
+        ram_mask(fuc, 0x100778, 0x00000700, data);
+
+        data = nv_ro16(bios, timing + 0x2c);
+        ram_mask(fuc, 0x10f250, 0x000003f0, (data & 0x003f) <<  4);
+        ram_mask(fuc, 0x10f24c, 0x7f000000, (data & 0x1fc0) << 18);
+
+        data = nv_ro08(bios, timing + 0x30);
+        ram_mask(fuc, 0x10f224, 0x001f0000, (data & 0xf8) << 13);
+
+        data = nv_ro16(bios, timing + 0x31);
+        ram_mask(fuc, 0x10fec4, 0x041e0f07, (data & 0x0800) << 15 |
+                                            (data & 0x0780) << 10 |
+                                            (data & 0x0078) <<  5 |
+                                            (data & 0x0007));
+        ram_mask(fuc, 0x10fec8, 0x00000027, (data & 0x8000) >> 10 |
+                                            (data & 0x7000) >> 12);
+
+        ram_wr32(fuc, 0x10f090, 0x4000007e);
+        ram_nsec(fuc, 1000);
+        ram_wr32(fuc, 0x10f314, 0x00000001); /* PRECHARGE */
+        ram_wr32(fuc, 0x10f310, 0x00000001); /* REFRESH */
+        ram_nsec(fuc, 2000);
+        ram_wr32(fuc, 0x10f210, 0x80000000); /* REFRESH_AUTO = 1 */
+
+        if ((nv_ro08(bios, ramcfg + 0x08) & 0x10) && (ram->mode == 2) /*XXX*/) {
+                u32 temp = ram_mask(fuc, 0x10f294, 0xff000000, 0x24000000);
+                train(fuc, 0xa4010000); /*XXX*/
+                ram_nsec(fuc, 1000);
+                ram_wr32(fuc, 0x10f294, temp);
+        }
+
+        ram_mask(fuc, mr[3], 0xfff, ram->base.mr[3]);
+        ram_wr32(fuc, mr[0], ram->base.mr[0]);
+        ram_mask(fuc, mr[8], 0xfff, ram->base.mr[8]);
+        ram_nsec(fuc, 1000);
+        ram_mask(fuc, mr[1], 0xfff, ram->base.mr[1]);
+        ram_mask(fuc, mr[5], 0xfff, ram->base.mr[5]);
+        ram_mask(fuc, mr[6], 0xfff, ram->base.mr[6]);
+        ram_mask(fuc, mr[7], 0xfff, ram->base.mr[7]);
+
+        if (vc == 0 && ram_have(fuc, gpio2E)) {
+                u32 temp  = ram_mask(fuc, gpio2E, 0x3000, fuc->r_func2E[0]);
+                if (temp != ram_rd32(fuc, gpio2E)) {
+                        ram_wr32(fuc, gpiotrig, 1);
+                        ram_nsec(fuc, 20000);
+                }
+        }
+
+        ram_mask(fuc, 0x10f200, 0x80000000, 0x80000000);
+        ram_wr32(fuc, 0x10f318, 0x00000001); /* NOP? */
+        ram_mask(fuc, 0x10f200, 0x80000000, 0x00000000);
+        ram_nsec(fuc, 1000);
+
+        data  = ram_rd32(fuc, 0x10f978);
+        data &= ~0x00046144;
+        data |=  0x0000000b;
+        if (!(nv_ro08(bios, ramcfg + 0x07) & 0x08)) {
+                if (!(nv_ro08(bios, ramcfg + 0x07) & 0x04))
+                        data |= 0x0000200c;
+                else
+                        data |= 0x00000000;
+        } else {
+                data |= 0x00040044;
+        }
+        ram_wr32(fuc, 0x10f978, data);
+
+        if (ram->mode == 1) {
+                data = ram_rd32(fuc, 0x10f830) | 0x00000001;
+                ram_wr32(fuc, 0x10f830, data);
+        }
+
+        if (!(nv_ro08(bios, ramcfg + 0x07) & 0x08)) {
+                data = 0x88020000;
+                if ( (nv_ro08(bios, ramcfg + 0x07) & 0x04))
+                        data |= 0x10000000;
+                if (!(nv_ro08(bios, rammap + 0x08) & 0x10))
+                        data |= 0x00080000;
+        } else {
+                data = 0xa40e0000;
+        }
+        train(fuc, data);
+        ram_nsec(fuc, 1000);
+
+        if (ram->mode == 2) { /*XXX*/
+                ram_mask(fuc, 0x10f800, 0x00000004, 0x00000004);
+        }
+
+        /* MR5: (re)enable LP3 if necessary
+         * XXX: need to find the switch, keeping off for now
+         */
+        ram_mask(fuc, mr[5], 0x00000004, 0x00000000);
+
+        if (ram->mode != 2) {
+                ram_mask(fuc, 0x10f830, 0x01000000, 0x01000000);
+                ram_mask(fuc, 0x10f830, 0x01000000, 0x00000000);
+        }
+
+        if (nv_ro08(bios, ramcfg + 0x07) & 0x02) {
+                ram_mask(fuc, 0x10f910, 0x80020000, 0x01000000);
+                ram_mask(fuc, 0x10f914, 0x80020000, 0x01000000);
+        }
+
+        ram_wr32(fuc, 0x62c000, 0x0f0f0f00);
+
+        if (nv_ro08(bios, rammap + 0x08) & 0x01)
+                data = 0x00000800;
+        else
+                data = 0x00000000;
+        ram_mask(fuc, 0x10f200, 0x00000800, data);
+        return 0;
+}
+
+/*******************************************************************************
+ * DDR3
+ ******************************************************************************/
+
+static int
+nve0_ram_calc_sddr3(struct nouveau_fb *pfb, u32 freq)
+{
+        struct nouveau_bios *bios = nouveau_bios(pfb);
+        struct nve0_ram *ram = (void *)pfb->ram;
+        struct nve0_ramfuc *fuc = &ram->fuc;
+        const u32 rcoef = ((  ram->P1 << 16) | (ram->N1 << 8) | ram->M1);
+        const u32 runk0 = ram->fN1 << 16;
+        const u32 runk1 = ram->fN1;
+        const u32 rammap = ram->base.rammap.data;
+        const u32 ramcfg = ram->base.ramcfg.data;
+        const u32 timing = ram->base.timing.data;
+        int vc = !(nv_ro08(bios, ramcfg + 0x02) & 0x08);
+        int mv = 1; /*XXX*/
+        u32 mask, data;
+
+        ram_mask(fuc, 0x10f808, 0x40000000, 0x40000000);
+        ram_wr32(fuc, 0x62c000, 0x0f0f0000);
+
+        if (vc == 1 && ram_have(fuc, gpio2E)) {
+                u32 temp  = ram_mask(fuc, gpio2E, 0x3000, fuc->r_func2E[1]);
+                if (temp != ram_rd32(fuc, gpio2E)) {
+                        ram_wr32(fuc, gpiotrig, 1);
+                        ram_nsec(fuc, 20000);
+                }
+        }
+
+        ram_mask(fuc, 0x10f200, 0x00000800, 0x00000000);
+        if ((nv_ro08(bios, ramcfg + 0x03) & 0xf0))
+                ram_mask(fuc, 0x10f808, 0x04000000, 0x04000000);
+
+        ram_wr32(fuc, 0x10f314, 0x00000001); /* PRECHARGE */
+        ram_wr32(fuc, 0x10f210, 0x00000000); /* REFRESH_AUTO = 0 */
+        ram_wr32(fuc, 0x10f310, 0x00000001); /* REFRESH */
+        ram_mask(fuc, 0x10f200, 0x80000000, 0x80000000);
+        ram_wr32(fuc, 0x10f310, 0x00000001); /* REFRESH */
+        ram_mask(fuc, 0x10f200, 0x80000000, 0x00000000);
+        ram_nsec(fuc, 1000);
+
+        ram_wr32(fuc, 0x10f090, 0x00000060);
+        ram_wr32(fuc, 0x10f090, 0xc000007e);
+
+        /*XXX: there does appear to be some kind of condition here, simply
+         *     modifying these bits in the vbios from the default pl0
+         *     entries shows no change.  however, the data does appear to
+         *     be correct and may be required for the transition back
+         */
+        mask = 0x00010000;
+        data = 0x00010000;
+
+        if (1) {
+                mask |= 0x800807e0;
+                data |= 0x800807e0;
+                switch (nv_ro08(bios, ramcfg + 0x03) & 0xc0) {
+                case 0xc0: data &= ~0x00000040; break;
+                case 0x80: data &= ~0x00000100; break;
+                case 0x40: data &= ~0x80000000; break;
+                case 0x00: data &= ~0x00000400; break;
+                }
+
+                switch (nv_ro08(bios, ramcfg + 0x03) & 0x30) {
+                case 0x30: data &= ~0x00000020; break;
+                case 0x20: data &= ~0x00000080; break;
+                case 0x10: data &= ~0x00080000; break;
+                case 0x00: data &= ~0x00000200; break;
+                }
+        }
+
+        if (nv_ro08(bios, ramcfg + 0x02) & 0x80)
+                mask |= 0x03000000;
+        if (nv_ro08(bios, ramcfg + 0x02) & 0x40)
+                mask |= 0x00002000;
+        if (nv_ro08(bios, ramcfg + 0x07) & 0x10)
+                mask |= 0x00004000;
+        if (nv_ro08(bios, ramcfg + 0x07) & 0x08)
+                mask |= 0x00000003;
+        else
+                mask |= 0x14000000;
+        ram_mask(fuc, 0x10f824, mask, data);
+
+        ram_mask(fuc, 0x132040, 0x00010000, 0x00000000);
+
+        ram_mask(fuc, 0x1373f4, 0x00000000, 0x00010010);
+        data  = ram_rd32(fuc, 0x1373ec) & ~0x00030000;
+        data |= (nv_ro08(bios, ramcfg + 0x03) & 0x30) << 12;
+        ram_wr32(fuc, 0x1373ec, data);
+        ram_mask(fuc, 0x1373f4, 0x00000003, 0x00000000);
+        ram_mask(fuc, 0x1373f4, 0x00000010, 0x00000000);
+
+        /* (re)program refpll, if required */
+        if ((ram_rd32(fuc, 0x132024) & 0xffffffff) != rcoef ||
+            (ram_rd32(fuc, 0x132034) & 0x0000ffff) != runk1) {
+                ram_mask(fuc, 0x132000, 0x00000001, 0x00000000);
+                ram_mask(fuc, 0x132020, 0x00000001, 0x00000000);
+                ram_wr32(fuc, 0x137320, 0x00000000);
+                ram_mask(fuc, 0x132030, 0xffff0000, runk0);
+                ram_mask(fuc, 0x132034, 0x0000ffff, runk1);
+                ram_wr32(fuc, 0x132024, rcoef);
+                ram_mask(fuc, 0x132028, 0x00080000, 0x00080000);
+                ram_mask(fuc, 0x132020, 0x00000001, 0x00000001);
+                ram_wait(fuc, 0x137390, 0x00020000, 0x00020000, 64000);
+                ram_mask(fuc, 0x132028, 0x00080000, 0x00000000);
+        }
+
+        ram_mask(fuc, 0x1373f4, 0x00000010, 0x00000010);
+        ram_mask(fuc, 0x1373f4, 0x00000003, 0x00000001);
+        ram_mask(fuc, 0x1373f4, 0x00010000, 0x00000000);
+
+        if (ram_have(fuc, gpioMV)) {
+                u32 temp  = ram_mask(fuc, gpioMV, 0x3000, fuc->r_funcMV[mv]);
+                if (temp != ram_rd32(fuc, gpioMV)) {
+                        ram_wr32(fuc, gpiotrig, 1);
+                        ram_nsec(fuc, 64000);
+                }
+        }
+
+        if ( (nv_ro08(bios, ramcfg + 0x02) & 0x40) ||
+             (nv_ro08(bios, ramcfg + 0x07) & 0x10)) {
+                ram_mask(fuc, 0x132040, 0x00010000, 0x00010000);
+                ram_nsec(fuc, 20000);
+        }
+
+        if (ram->mode != 2) /*XXX*/ {
+                if (nv_ro08(bios, ramcfg + 0x07) & 0x40)
+                        ram_mask(fuc, 0x10f670, 0x80000000, 0x80000000);
+        }
+
+        data = (nv_ro08(bios, rammap + 0x11) & 0x0c) >> 2;
+        ram_wr32(fuc, 0x10f65c, 0x00000011 * data);
+        ram_wr32(fuc, 0x10f6b8, 0x01010101 * nv_ro08(bios, ramcfg + 0x09));
+        ram_wr32(fuc, 0x10f6bc, 0x01010101 * nv_ro08(bios, ramcfg + 0x09));
+
+        mask = 0x00010000;
+        data = 0x00000000;
+        if (!(nv_ro08(bios, ramcfg + 0x02) & 0x80))
+                data |= 0x03000000;
+        if (!(nv_ro08(bios, ramcfg + 0x02) & 0x40))
+                data |= 0x00002000;
+        if (!(nv_ro08(bios, ramcfg + 0x07) & 0x10))
+                data |= 0x00004000;
+        if (!(nv_ro08(bios, ramcfg + 0x07) & 0x08))
+                data |= 0x00000003;
+        else
+                data |= 0x14000000;
+        ram_mask(fuc, 0x10f824, mask, data);
+        ram_nsec(fuc, 1000);
+
+        if (nv_ro08(bios, ramcfg + 0x08) & 0x01)
+                data = 0x00100000;
+        else
+                data = 0x00000000;
+        ram_mask(fuc, 0x10f82c, 0x00100000, data);
+
+        /* PFB timing */
+        ram_mask(fuc, 0x10f248, 0xffffffff, nv_ro32(bios, timing + 0x28));
+        ram_mask(fuc, 0x10f290, 0xffffffff, nv_ro32(bios, timing + 0x00));
+        ram_mask(fuc, 0x10f294, 0xffffffff, nv_ro32(bios, timing + 0x04));
+        ram_mask(fuc, 0x10f298, 0xffffffff, nv_ro32(bios, timing + 0x08));
+        ram_mask(fuc, 0x10f29c, 0xffffffff, nv_ro32(bios, timing + 0x0c));
+        ram_mask(fuc, 0x10f2a0, 0xffffffff, nv_ro32(bios, timing + 0x10));
+        ram_mask(fuc, 0x10f2a4, 0xffffffff, nv_ro32(bios, timing + 0x14));
+        ram_mask(fuc, 0x10f2a8, 0xffffffff, nv_ro32(bios, timing + 0x18));
+        ram_mask(fuc, 0x10f2ac, 0xffffffff, nv_ro32(bios, timing + 0x1c));
+        ram_mask(fuc, 0x10f2cc, 0xffffffff, nv_ro32(bios, timing + 0x20));
+        ram_mask(fuc, 0x10f2e8, 0xffffffff, nv_ro32(bios, timing + 0x24));
+
+        mask = 0x33f00000;
+        data = 0x00000000;
+        if (!(nv_ro08(bios, ramcfg + 0x01) & 0x04))
+                data |= 0x20200000;
+        if (!(nv_ro08(bios, ramcfg + 0x07) & 0x80))
+                data |= 0x12800000;
+        /*XXX: see note above about there probably being some condition
+         *     for the 10f824 stuff that uses ramcfg 3...
+         */
+        if ( (nv_ro08(bios, ramcfg + 0x03) & 0xf0)) {
+                if (nv_ro08(bios, rammap + 0x08) & 0x0c) {
+                        if (!(nv_ro08(bios, ramcfg + 0x07) & 0x80))
+                                mask |= 0x00000020;
+                        else
+                                data |= 0x00000020;
+                        mask |= 0x08000004;
+                }
+                data |= 0x04000000;
+        } else {
+                mask |= 0x44000020;
+                data |= 0x08000004;
+        }
+
+        ram_mask(fuc, 0x10f808, mask, data);
+
+        data = nv_ro08(bios, ramcfg + 0x03) & 0x0f;
+        ram_wr32(fuc, 0x10f870, 0x11111111 * data);
+
+        data = nv_ro16(bios, timing + 0x2c);
+        ram_mask(fuc, 0x10f250, 0x000003f0, (data & 0x003f) <<  4);
+
+        if (((nv_ro32(bios, timing + 0x2c) & 0x00001fc0) >>  6) >
+            ((nv_ro32(bios, timing + 0x28) & 0x7f000000) >> 24))
+                data = (nv_ro32(bios, timing + 0x2c) & 0x00001fc0) >>  6;
+        else
+                data = (nv_ro32(bios, timing + 0x28) & 0x1f000000) >> 24;
+        ram_mask(fuc, 0x10f24c, 0x7f000000, data << 24);
+
+        data = nv_ro08(bios, timing + 0x30);
+        ram_mask(fuc, 0x10f224, 0x001f0000, (data & 0xf8) << 13);
+
+        ram_wr32(fuc, 0x10f090, 0x4000007f);
+        ram_nsec(fuc, 1000);
+
+        ram_wr32(fuc, 0x10f314, 0x00000001); /* PRECHARGE */
+        ram_wr32(fuc, 0x10f310, 0x00000001); /* REFRESH */
+        ram_wr32(fuc, 0x10f210, 0x80000000); /* REFRESH_AUTO = 1 */
+        ram_nsec(fuc, 1000);
+
+        ram_nuke(fuc, mr[0]);
+        ram_mask(fuc, mr[0], 0x100, 0x100);
+        ram_mask(fuc, mr[0], 0x100, 0x000);
+
+        ram_mask(fuc, mr[2], 0xfff, ram->base.mr[2]);
+        ram_wr32(fuc, mr[0], ram->base.mr[0]);
+        ram_nsec(fuc, 1000);
+
+        ram_nuke(fuc, mr[0]);
+        ram_mask(fuc, mr[0], 0x100, 0x100);
+        ram_mask(fuc, mr[0], 0x100, 0x000);
+
+        if (vc == 0 && ram_have(fuc, gpio2E)) {
+                u32 temp  = ram_mask(fuc, gpio2E, 0x3000, fuc->r_func2E[0]);
+                if (temp != ram_rd32(fuc, gpio2E)) {
+                        ram_wr32(fuc, gpiotrig, 1);
+                        ram_nsec(fuc, 20000);
+                }
+        }
+
+        if (ram->mode != 2) {
+                ram_mask(fuc, 0x10f830, 0x01000000, 0x01000000);
+                ram_mask(fuc, 0x10f830, 0x01000000, 0x00000000);
+        }
+
+        ram_mask(fuc, 0x10f200, 0x80000000, 0x80000000);
+        ram_wr32(fuc, 0x10f318, 0x00000001); /* NOP? */
+        ram_mask(fuc, 0x10f200, 0x80000000, 0x00000000);
+        ram_nsec(fuc, 1000);
+
+        ram_wr32(fuc, 0x62c000, 0x0f0f0f00);
+
+        if (nv_ro08(bios, rammap + 0x08) & 0x01)
+                data = 0x00000800;
+        else
+                data = 0x00000000;
+        ram_mask(fuc, 0x10f200, 0x00000800, data);
+        return 0;
+}
+
+/*******************************************************************************
+ * main hooks
+ ******************************************************************************/
+
+static int
+nve0_ram_calc(struct nouveau_fb *pfb, u32 freq)
+{
+        struct nouveau_bios *bios = nouveau_bios(pfb);
+        struct nve0_ram *ram = (void *)pfb->ram;
+        struct nve0_ramfuc *fuc = &ram->fuc;
+        struct bit_entry M;
+        int ret, refclk, strap, i;
+        u32 data;
+        u8  cnt;
+
+        /* lookup memory config data relevant to the target frequency */
+        ram->base.rammap.data = nvbios_rammap_match(bios, freq / 1000,
+                                                   &ram->base.rammap.version,
+                                                   &ram->base.rammap.size, &cnt,
+                                                   &ram->base.ramcfg.size);
+        if (!ram->base.rammap.data || ram->base.rammap.version != 0x11 ||
+             ram->base.rammap.size < 0x09) {
+                nv_error(pfb, "invalid/missing rammap entry\n");
+                return -EINVAL;
+        }
+
+        /* locate specific data set for the attached memory */
+        if (bit_entry(bios, 'M', &M) || M.version != 2 || M.length < 3) {
+                nv_error(pfb, "invalid/missing memory table\n");
+                return -EINVAL;
+        }
+
+        strap = (nv_rd32(pfb, 0x101000) & 0x0000003c) >> 2;
+        data = nv_ro16(bios, M.offset + 1);
+        if (data)
+                strap = nv_ro08(bios, data + strap);
+
+        if (strap >= cnt) {
+                nv_error(pfb, "invalid ramcfg strap\n");
+                return -EINVAL;
+        }
+
+        ram->base.ramcfg.version = ram->base.rammap.version;
+        ram->base.ramcfg.data = ram->base.rammap.data + ram->base.rammap.size +
+                               (ram->base.ramcfg.size * strap);
+        if (!ram->base.ramcfg.data || ram->base.ramcfg.version != 0x11 ||
+             ram->base.ramcfg.size < 0x08) {
+                nv_error(pfb, "invalid/missing ramcfg entry\n");
+                return -EINVAL;
+        }
+
+        /* lookup memory timings, if bios says they're present */
+        strap = nv_ro08(bios, ram->base.ramcfg.data + 0x00);
+        if (strap != 0xff) {
+                ram->base.timing.data =
+                        nvbios_timing_entry(bios, strap,
+                                           &ram->base.timing.version,
+                                           &ram->base.timing.size);
+                if (!ram->base.timing.data ||
+                     ram->base.timing.version != 0x20 ||
+                     ram->base.timing.size < 0x33) {
+                        nv_error(pfb, "invalid/missing timing entry\n");
+                        return -EINVAL;
+                }
+        } else {
+                ram->base.timing.data = 0;
+        }
+
+        ret = ram_init(fuc, pfb);
+        if (ret)
+                return ret;
+
+        ram->mode = (freq > fuc->refpll.vco1.max_freq) ? 2 : 1;
+        ram->from = ram_rd32(fuc, 0x1373f4) & 0x0000000f;
+
+        /* XXX: this is *not* what nvidia do.  on fermi nvidia generally
+         * select, based on some unknown condition, one of the two possible
+         * reference frequencies listed in the vbios table for mempll and
+         * program refpll to that frequency.
+         *
+         * so far, i've seen very weird values being chosen by nvidia on
+         * kepler boards, no idea how/why they're chosen.
+         */
+        refclk = freq;
+        if (ram->mode == 2)
+                refclk = fuc->mempll.refclk;
+
+        /* calculate refpll coefficients */
+        ret = nva3_pll_calc(nv_subdev(pfb), &fuc->refpll, refclk, &ram->N1,
+                           &ram->fN1, &ram->M1, &ram->P1);
+        fuc->mempll.refclk = ret;
+        if (ret <= 0) {
+                nv_error(pfb, "unable to calc refpll\n");
+                return -EINVAL;
+        }
+
+        /* calculate mempll coefficients, if we're using it */
+        if (ram->mode == 2) {
+                /* post-divider doesn't work... the reg takes the values but
+                 * appears to completely ignore it.  there *is* a bit at
+                 * bit 28 that appears to divide the clock by 2 if set.
+                 */
+                fuc->mempll.min_p = 1;
+                fuc->mempll.max_p = 2;
+
+                ret = nva3_pll_calc(nv_subdev(pfb), &fuc->mempll, freq,
+                                   &ram->N2, NULL, &ram->M2, &ram->P2);
+                if (ret <= 0) {
+                        nv_error(pfb, "unable to calc mempll\n");
+                        return -EINVAL;
+                }
+        }
+
+        for (i = 0; i < ARRAY_SIZE(fuc->r_mr); i++) {
+                if (ram_have(fuc, mr[i]))
+                        ram->base.mr[i] = ram_rd32(fuc, mr[i]);
+        }
+
+        switch (ram->base.type) {
+        case NV_MEM_TYPE_DDR3:
+                ret = nouveau_sddr3_calc(&ram->base);
+                if (ret == 0)
+                        ret = nve0_ram_calc_sddr3(pfb, freq);
+                break;
+        case NV_MEM_TYPE_GDDR5:
+                ret = nouveau_gddr5_calc(&ram->base);
+                if (ret == 0)
+                        ret = nve0_ram_calc_gddr5(pfb, freq);
+                break;
+        default:
+                ret = -ENOSYS;
+                break;
+        }
+
+        return ret;
+}
+
+static int
+nve0_ram_prog(struct nouveau_fb *pfb)
+{
+        struct nouveau_device *device = nv_device(pfb);
+        struct nve0_ram *ram = (void *)pfb->ram;
+        struct nve0_ramfuc *fuc = &ram->fuc;
+        ram_exec(fuc, nouveau_boolopt(device->cfgopt, "NvMemExec", false));
+        return 0;
+}
+
+static void
+nve0_ram_tidy(struct nouveau_fb *pfb)
+{
+        struct nve0_ram *ram = (void *)pfb->ram;
+        struct nve0_ramfuc *fuc = &ram->fuc;
+        ram_exec(fuc, false);
+}
+
+static int
+nve0_ram_init(struct nouveau_object *object)
+{
+        struct nouveau_fb *pfb = (void *)object->parent;
+        struct nve0_ram *ram   = (void *)object;
+        struct nouveau_bios *bios = nouveau_bios(pfb);
+        static const u8  train0[] = {
+                0x00, 0xff, 0xff, 0x00, 0xff, 0x00,
+                0x00, 0xff, 0xff, 0x00, 0xff, 0x00,
+        };
+        static const u32 train1[] = {
+                0x00000000, 0xffffffff,
+                0x55555555, 0xaaaaaaaa,
+                0x33333333, 0xcccccccc,
+                0xf0f0f0f0, 0x0f0f0f0f,
+                0x00ff00ff, 0xff00ff00,
+                0x0000ffff, 0xffff0000,
+        };
+        u8  ver, hdr, cnt, len, snr, ssz;
+        u32 data, save;
+        int ret, i;
+
+        ret = nouveau_ram_init(&ram->base);
+        if (ret)
+                return ret;
+
+        /* run a bunch of tables from rammap table.  there's actually
+         * individual pointers for each rammap entry too, but, nvidia
+         * seem to just run the last two entries' scripts early on in
+         * their init, and never again.. we'll just run 'em all once
+         * for now.
+         *
+         * i strongly suspect that each script is for a separate mode
+         * (likely selected by 0x10f65c's lower bits?), and the
+         * binary driver skips the one that's already been setup by
+         * the init tables.
+         */
+        data = nvbios_rammap_table(bios, &ver, &hdr, &cnt, &len, &snr, &ssz);
+        if (!data || hdr < 0x15)
+                return -EINVAL;
+
+        cnt  = nv_ro08(bios, data + 0x14); /* guess at count */
+        data = nv_ro32(bios, data + 0x10); /* guess u32... */
+        save = nv_rd32(pfb, 0x10f65c);
+        for (i = 0; i < cnt; i++) {
+                nv_mask(pfb, 0x10f65c, 0x000000f0, i << 4);
+                nvbios_exec(&(struct nvbios_init) {
+                                .subdev = nv_subdev(pfb),
+                                .bios = bios,
+                                .offset = nv_ro32(bios, data), /* guess u32 */
+                                .execute = 1,
+                            });
+                data += 4;
+        }
+        nv_wr32(pfb, 0x10f65c, save);
+
+        switch (ram->base.type) {
+        case NV_MEM_TYPE_GDDR5:
+                for (i = 0; i < 0x30; i++) {
+                        nv_wr32(pfb, 0x10f968, 0x00000000 | (i << 8));
+                        nv_wr32(pfb, 0x10f920, 0x00000000 | train0[i % 12]);
+                        nv_wr32(pfb, 0x10f918,              train1[i % 12]);
+                        nv_wr32(pfb, 0x10f920, 0x00000100 | train0[i % 12]);
+                        nv_wr32(pfb, 0x10f918,              train1[i % 12]);
+
+                        nv_wr32(pfb, 0x10f96c, 0x00000000 | (i << 8));
+                        nv_wr32(pfb, 0x10f924, 0x00000000 | train0[i % 12]);
+                        nv_wr32(pfb, 0x10f91c,              train1[i % 12]);
+                        nv_wr32(pfb, 0x10f924, 0x00000100 | train0[i % 12]);
+                        nv_wr32(pfb, 0x10f91c,              train1[i % 12]);
+                }
+
+                for (i = 0; i < 0x100; i++) {
+                        nv_wr32(pfb, 0x10f968, i);
+                        nv_wr32(pfb, 0x10f900, train1[2 + (i & 1)]);
+                }
+
+                for (i = 0; i < 0x100; i++) {
+                        nv_wr32(pfb, 0x10f96c, i);
+                        nv_wr32(pfb, 0x10f900, train1[2 + (i & 1)]);
+                }
+                break;
+        default:
+                break;
+        }
+
+        return 0;
+}
+
+static int
+nve0_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+              struct nouveau_oclass *oclass, void *data, u32 size,
+              struct nouveau_object **pobject)
+{
+        struct nouveau_fb *pfb = nouveau_fb(parent);
+        struct nouveau_bios *bios = nouveau_bios(pfb);
+        struct nouveau_gpio *gpio = nouveau_gpio(pfb);
+        struct dcb_gpio_func func;
+        struct nve0_ram *ram;
+        int ret;
+
+        ret = nvc0_ram_create(parent, engine, oclass, &ram);
+        *pobject = nv_object(ram);
+        if (ret)
+                return ret;
+
+        switch (ram->base.type) {
+        case NV_MEM_TYPE_DDR3:
+        case NV_MEM_TYPE_GDDR5:
+                ram->base.calc = nve0_ram_calc;
+                ram->base.prog = nve0_ram_prog;
+                ram->base.tidy = nve0_ram_tidy;
+                break;
+        default:
+                nv_warn(pfb, "reclocking of this RAM type is unsupported\n");
+                break;
+        }
+
+        // parse bios data for both pll's
+        ret = nvbios_pll_parse(bios, 0x0c, &ram->fuc.refpll);
+        if (ret) {
+                nv_error(pfb, "mclk refpll data not found\n");
+                return ret;
+        }
+
+        ret = nvbios_pll_parse(bios, 0x04, &ram->fuc.mempll);
+        if (ret) {
+                nv_error(pfb, "mclk pll data not found\n");
+                return ret;
+        }
+
+        ret = gpio->find(gpio, 0, 0x18, DCB_GPIO_UNUSED, &func);
+        if (ret == 0) {
+                ram->fuc.r_gpioMV = ramfuc_reg(0x00d610 + (func.line * 0x04));
+                ram->fuc.r_funcMV[0] = (func.log[0] ^ 2) << 12;
+                ram->fuc.r_funcMV[1] = (func.log[1] ^ 2) << 12;
+        }
+
+        ret = gpio->find(gpio, 0, 0x2e, DCB_GPIO_UNUSED, &func);
+        if (ret == 0) {
+                ram->fuc.r_gpio2E = ramfuc_reg(0x00d610 + (func.line * 0x04));
+                ram->fuc.r_func2E[0] = (func.log[0] ^ 2) << 12;
+                ram->fuc.r_func2E[1] = (func.log[1] ^ 2) << 12;
+        }
+
+        ram->fuc.r_gpiotrig = ramfuc_reg(0x00d604);
+
+        ram->fuc.r_0x132020 = ramfuc_reg(0x132020);
+        ram->fuc.r_0x132028 = ramfuc_reg(0x132028);
+        ram->fuc.r_0x132024 = ramfuc_reg(0x132024);
+        ram->fuc.r_0x132030 = ramfuc_reg(0x132030);
+        ram->fuc.r_0x132034 = ramfuc_reg(0x132034);
+        ram->fuc.r_0x132000 = ramfuc_reg(0x132000);
+        ram->fuc.r_0x132004 = ramfuc_reg(0x132004);
+        ram->fuc.r_0x132040 = ramfuc_reg(0x132040);
+
+        ram->fuc.r_0x10f248 = ramfuc_reg(0x10f248);
+        ram->fuc.r_0x10f290 = ramfuc_reg(0x10f290);
+        ram->fuc.r_0x10f294 = ramfuc_reg(0x10f294);
+        ram->fuc.r_0x10f298 = ramfuc_reg(0x10f298);
+        ram->fuc.r_0x10f29c = ramfuc_reg(0x10f29c);
+        ram->fuc.r_0x10f2a0 = ramfuc_reg(0x10f2a0);
+        ram->fuc.r_0x10f2a4 = ramfuc_reg(0x10f2a4);
+        ram->fuc.r_0x10f2a8 = ramfuc_reg(0x10f2a8);
+        ram->fuc.r_0x10f2ac = ramfuc_reg(0x10f2ac);
+        ram->fuc.r_0x10f2cc = ramfuc_reg(0x10f2cc);
+        ram->fuc.r_0x10f2e8 = ramfuc_reg(0x10f2e8);
+        ram->fuc.r_0x10f250 = ramfuc_reg(0x10f250);
+        ram->fuc.r_0x10f24c = ramfuc_reg(0x10f24c);
+        ram->fuc.r_0x10fec4 = ramfuc_reg(0x10fec4);
+        ram->fuc.r_0x10fec8 = ramfuc_reg(0x10fec8);
+        ram->fuc.r_0x10f604 = ramfuc_reg(0x10f604);
+        ram->fuc.r_0x10f614 = ramfuc_reg(0x10f614);
+        ram->fuc.r_0x10f610 = ramfuc_reg(0x10f610);
+        ram->fuc.r_0x100770 = ramfuc_reg(0x100770);
+        ram->fuc.r_0x100778 = ramfuc_reg(0x100778);
+        ram->fuc.r_0x10f224 = ramfuc_reg(0x10f224);
+
+        ram->fuc.r_0x10f870 = ramfuc_reg(0x10f870);
+        ram->fuc.r_0x10f698 = ramfuc_reg(0x10f698);
+        ram->fuc.r_0x10f694 = ramfuc_reg(0x10f694);
+        ram->fuc.r_0x10f6b8 = ramfuc_reg(0x10f6b8);
+        ram->fuc.r_0x10f808 = ramfuc_reg(0x10f808);
+        ram->fuc.r_0x10f670 = ramfuc_reg(0x10f670);
+        ram->fuc.r_0x10f60c = ramfuc_reg(0x10f60c);
+        ram->fuc.r_0x10f830 = ramfuc_reg(0x10f830);
+        ram->fuc.r_0x1373ec = ramfuc_reg(0x1373ec);
+        ram->fuc.r_0x10f800 = ramfuc_reg(0x10f800);
+        ram->fuc.r_0x10f82c = ramfuc_reg(0x10f82c);
+
+        ram->fuc.r_0x10f978 = ramfuc_reg(0x10f978);
+        ram->fuc.r_0x10f910 = ramfuc_reg(0x10f910);
+        ram->fuc.r_0x10f914 = ramfuc_reg(0x10f914);
+
+        switch (ram->base.type) {
+        case NV_MEM_TYPE_GDDR5:
+                ram->fuc.r_mr[0] = ramfuc_reg(0x10f300);
+                ram->fuc.r_mr[1] = ramfuc_reg(0x10f330);
+                ram->fuc.r_mr[2] = ramfuc_reg(0x10f334);
+                ram->fuc.r_mr[3] = ramfuc_reg(0x10f338);
+                ram->fuc.r_mr[4] = ramfuc_reg(0x10f33c);
+                ram->fuc.r_mr[5] = ramfuc_reg(0x10f340);
+                ram->fuc.r_mr[6] = ramfuc_reg(0x10f344);
+                ram->fuc.r_mr[7] = ramfuc_reg(0x10f348);
+                ram->fuc.r_mr[8] = ramfuc_reg(0x10f354);
+                ram->fuc.r_mr[15] = ramfuc_reg(0x10f34c);
+                break;
+        case NV_MEM_TYPE_DDR3:
+                ram->fuc.r_mr[0] = ramfuc_reg(0x10f300);
+                ram->fuc.r_mr[2] = ramfuc_reg(0x10f320);
+                break;
+        default:
+                break;
+        }
+
+        ram->fuc.r_0x62c000 = ramfuc_reg(0x62c000);
+        ram->fuc.r_0x10f200 = ramfuc_reg(0x10f200);
+        ram->fuc.r_0x10f210 = ramfuc_reg(0x10f210);
+        ram->fuc.r_0x10f310 = ramfuc_reg(0x10f310);
+        ram->fuc.r_0x10f314 = ramfuc_reg(0x10f314);
+        ram->fuc.r_0x10f318 = ramfuc_reg(0x10f318);
+        ram->fuc.r_0x10f090 = ramfuc_reg(0x10f090);
+        ram->fuc.r_0x10f69c = ramfuc_reg(0x10f69c);
+        ram->fuc.r_0x10f824 = ramfuc_reg(0x10f824);
+        ram->fuc.r_0x1373f0 = ramfuc_reg(0x1373f0);
+        ram->fuc.r_0x1373f4 = ramfuc_reg(0x1373f4);
+        ram->fuc.r_0x137320 = ramfuc_reg(0x137320);
+        ram->fuc.r_0x10f65c = ramfuc_reg(0x10f65c);
+        ram->fuc.r_0x10f6bc = ramfuc_reg(0x10f6bc);
+        ram->fuc.r_0x100710 = ramfuc_reg(0x100710);
+        ram->fuc.r_0x10f750 = ramfuc_reg(0x10f750);
+        return 0;
+}
+
+struct nouveau_oclass
+nve0_ram_oclass = {
+        .handle = 0,
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nve0_ram_ctor,
+                .dtor = _nouveau_ram_dtor,
+                .init = nve0_ram_init,
+                .fini = _nouveau_ram_fini,
+        }
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramseq.h b/drivers/gpu/drm/nouveau/core/subdev/fb/ramseq.h
new file mode 100644
index 000000000000..571077e39071
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramseq.h
@@ -0,0 +1,18 @@
+#ifndef __NVKM_FBRAM_SEQ_H__
+#define __NVKM_FBRAM_SEQ_H__
+
+#include <subdev/bus.h>
+#include <subdev/bus/hwsq.h>
+
+#define ram_init(s,p)       hwsq_init(&(s)->base, (p))
+#define ram_exec(s,e)       hwsq_exec(&(s)->base, (e))
+#define ram_have(s,r)       ((s)->r_##r.addr != 0x000000)
+#define ram_rd32(s,r)       hwsq_rd32(&(s)->base, &(s)->r_##r)
+#define ram_wr32(s,r,d)     hwsq_wr32(&(s)->base, &(s)->r_##r, (d))
+#define ram_nuke(s,r)       hwsq_nuke(&(s)->base, &(s)->r_##r)
+#define ram_mask(s,r,m,d)   hwsq_mask(&(s)->base, &(s)->r_##r, (m), (d))
+#define ram_setf(s,f,d)     hwsq_setf(&(s)->base, (f), (d))
+#define ram_wait(s,f,d)     hwsq_wait(&(s)->base, (f), (d))
+#define ram_nsec(s,n)       hwsq_nsec(&(s)->base, (n))
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/sddr3.c b/drivers/gpu/drm/nouveau/core/subdev/fb/sddr3.c
new file mode 100644
index 000000000000..ebd4cd9c35d9
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/sddr3.c
@@ -0,0 +1,99 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include <subdev/bios.h>
+#include "priv.h"
+
+struct ramxlat {
+        int id;
+        u8 enc;
+};
+
+static inline int
+ramxlat(const struct ramxlat *xlat, int id)
+{
+        while (xlat->id >= 0) {
+                if (xlat->id == id)
+                        return xlat->enc;
+                xlat++;
+        }
+        return -EINVAL;
+}
+
+static const struct ramxlat
+ramddr3_cl[] = {
+        { 5, 2 }, { 6, 4 }, { 7, 6 }, { 8, 8 }, { 9, 10 }, { 10, 12 },
+        { 11, 14 },
+        /* the below are mentioned in some, but not all, ddr3 docs */
+        { 12, 1 }, { 13, 3 }, { 14, 5 },
+        { -1 }
+};
+
+static const struct ramxlat
+ramddr3_wr[] = {
+        { 5, 1 }, { 6, 2 }, { 7, 3 }, { 8, 4 }, { 10, 5 }, { 12, 6 },
+        /* the below are mentioned in some, but not all, ddr3 docs */
+        { 14, 7 }, { 16, 0 },
+        { -1 }
+};
+
+static const struct ramxlat
+ramddr3_cwl[] = {
+        { 5, 0 }, { 6, 1 }, { 7, 2 }, { 8, 3 },
+        /* the below are mentioned in some, but not all, ddr3 docs */
+        { 9, 4 },
+        { -1 }
+};
+
+int
+nouveau_sddr3_calc(struct nouveau_ram *ram)
+{
+        struct nouveau_bios *bios = nouveau_bios(ram);
+        int WL, CL, WR;
+
+        switch (!!ram->timing.data * ram->timing.version) {
+        case 0x20:
+                WL = (nv_ro16(bios, ram->timing.data + 0x04) & 0x0f80) >> 7;
+                CL =  nv_ro08(bios, ram->timing.data + 0x04) & 0x1f;
+                WR =  nv_ro08(bios, ram->timing.data + 0x0a) & 0x7f;
+                break;
+        default:
+                return -ENOSYS;
+        }
+
+        WL = ramxlat(ramddr3_cwl, WL);
+        CL = ramxlat(ramddr3_cl, CL);
+        WR = ramxlat(ramddr3_wr, WR);
+        if (WL < 0 || CL < 0 || WR < 0)
+                return -EINVAL;
+
+        ram->mr[0] &= ~0xe74;
+        ram->mr[0] |= (WR & 0x07) << 9;
+        ram->mr[0] |= (CL & 0x0e) << 3;
+        ram->mr[0] |= (CL & 0x01) << 2;
+
+        ram->mr[2] &= ~0x038;
+        ram->mr[2] |= (WL & 0x07) << 3;
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/gpio/base.c b/drivers/gpu/drm/nouveau/core/subdev/gpio/base.c
index d422acc9af15..f572c2804c32 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/gpio/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/gpio/base.c
@@ -67,7 +67,7 @@ nouveau_gpio_find(struct nouveau_gpio *gpio, int idx, u8 tag, u8 line,
                 }
         }
 
-        return -EINVAL;
+        return -ENOENT;
 }
 
 static int
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
index 2895c19bb152..041fd5edaebf 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
@@ -195,7 +195,7 @@ nouveau_i2c_find_type(struct nouveau_i2c *i2c, u16 type)
 
 static int
 nouveau_i2c_identify(struct nouveau_i2c *i2c, int index, const char *what,
-                     struct i2c_board_info *info,
+                     struct nouveau_i2c_board_info *info,
                      bool (*match)(struct nouveau_i2c_port *,
                                    struct i2c_board_info *))
 {
@@ -208,12 +208,29 @@ nouveau_i2c_identify(struct nouveau_i2c *i2c, int index, const char *what,
         }
 
         nv_debug(i2c, "probing %ss on bus: %d\n", what, port->index);
-        for (i = 0; info[i].addr; i++) {
-                if (nv_probe_i2c(port, info[i].addr) &&
-                    (!match || match(port, &info[i]))) {
-                        nv_info(i2c, "detected %s: %s\n", what, info[i].type);
+        for (i = 0; info[i].dev.addr; i++) {
+                u8 orig_udelay = 0;
+
+                if ((port->adapter.algo == &i2c_bit_algo) &&
+                    (info[i].udelay != 0)) {
+                        struct i2c_algo_bit_data *algo = port->adapter.algo_data;
+                        nv_debug(i2c, "using custom udelay %d instead of %d\n",
+                                 info[i].udelay, algo->udelay);
+                        orig_udelay = algo->udelay;
+                        algo->udelay = info[i].udelay;
+                }
+
+                if (nv_probe_i2c(port, info[i].dev.addr) &&
+                    (!match || match(port, &info[i].dev))) {
+                        nv_info(i2c, "detected %s: %s\n", what,
+                                info[i].dev.type);
                         return i;
                 }
+
+                if (orig_udelay) {
+                        struct i2c_algo_bit_data *algo = port->adapter.algo_data;
+                        algo->udelay = orig_udelay;
+                }
         }
 
         nv_debug(i2c, "no devices found.\n");
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/base.c b/drivers/gpu/drm/nouveau/core/subdev/mc/base.c
index e290cfa4acee..b4b9943773bc 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/mc/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/base.c
@@ -25,38 +25,48 @@
 #include <subdev/mc.h>
 #include <core/option.h>
 
+static inline u32
+nouveau_mc_intr_mask(struct nouveau_mc *pmc)
+{
+        u32 intr = nv_rd32(pmc, 0x000100);
+        if (intr == 0xffffffff) /* likely fallen off the bus */
+                intr = 0x00000000;
+        return intr;
+}
+
 static irqreturn_t
 nouveau_mc_intr(int irq, void *arg)
 {
         struct nouveau_mc *pmc = arg;
-        const struct nouveau_mc_intr *map = pmc->intr_map;
-        struct nouveau_device *device = nv_device(pmc);
+        const struct nouveau_mc_oclass *oclass = (void *)nv_object(pmc)->oclass;
+        const struct nouveau_mc_intr *map = oclass->intr;
         struct nouveau_subdev *unit;
-        u32 stat, intr;
-
-        intr = stat = nv_rd32(pmc, 0x000100);
-        if (intr == 0xffffffff)
-                return IRQ_NONE;
-        while (stat && map->stat) {
-                if (stat & map->stat) {
-                        unit = nouveau_subdev(pmc, map->unit);
-                        if (unit && unit->intr)
-                                unit->intr(unit);
-                        intr &= ~map->stat;
-                }
-                map++;
-        }
+        u32 intr;
 
+        nv_wr32(pmc, 0x000140, 0x00000000);
+        nv_rd32(pmc, 0x000140);
+        intr = nouveau_mc_intr_mask(pmc);
         if (pmc->use_msi)
-                nv_wr08(pmc->base.base.parent, 0x00088068, 0xff);
+                oclass->msi_rearm(pmc);
 
         if (intr) {
-                nv_error(pmc, "unknown intr 0x%08x\n", stat);
+                u32 stat = intr = nouveau_mc_intr_mask(pmc);
+                while (map->stat) {
+                        if (intr & map->stat) {
+                                unit = nouveau_subdev(pmc, map->unit);
+                                if (unit && unit->intr)
+                                        unit->intr(unit);
+                                stat &= ~map->stat;
+                        }
+                        map++;
+                }
+
+                if (stat)
+                        nv_error(pmc, "unknown intr 0x%08x\n", stat);
         }
 
-        if (stat == IRQ_HANDLED)
-                pm_runtime_mark_last_busy(&device->pdev->dev);
-        return stat ? IRQ_HANDLED : IRQ_NONE;
+        nv_wr32(pmc, 0x000140, 0x00000001);
+        return intr ? IRQ_HANDLED : IRQ_NONE;
 }
 
 int
@@ -91,37 +101,42 @@ _nouveau_mc_dtor(struct nouveau_object *object)
 
 int
 nouveau_mc_create_(struct nouveau_object *parent, struct nouveau_object *engine,
-                   struct nouveau_oclass *oclass,
-                   const struct nouveau_mc_intr *intr_map,
-                   int length, void **pobject)
+                   struct nouveau_oclass *bclass, int length, void **pobject)
 {
+        const struct nouveau_mc_oclass *oclass = (void *)bclass;
         struct nouveau_device *device = nv_device(parent);
         struct nouveau_mc *pmc;
         int ret;
 
-        ret = nouveau_subdev_create_(parent, engine, oclass, 0, "PMC",
+        ret = nouveau_subdev_create_(parent, engine, bclass, 0, "PMC",
                                      "master", length, pobject);
         pmc = *pobject;
         if (ret)
                 return ret;
 
-        pmc->intr_map = intr_map;
-
         switch (device->pdev->device & 0x0ff0) {
-        case 0x00f0: /* BR02? */
-        case 0x02e0: /* BR02? */
-                pmc->use_msi = false;
+        case 0x00f0:
+        case 0x02e0:
+                /* BR02? NFI how these would be handled yet exactly */
                 break;
         default:
-                pmc->use_msi = nouveau_boolopt(device->cfgopt, "NvMSI", false);
+                switch (device->chipset) {
+                case 0xaa: break; /* reported broken, nv also disable it */
+                default:
+                        pmc->use_msi = true;
+                        break;
+                }
+        }
+
+        pmc->use_msi = nouveau_boolopt(device->cfgopt, "NvMSI", pmc->use_msi);
+        if (pmc->use_msi && oclass->msi_rearm) {
+                pmc->use_msi = pci_enable_msi(device->pdev) == 0;
                 if (pmc->use_msi) {
-                        pmc->use_msi = pci_enable_msi(device->pdev) == 0;
-                        if (pmc->use_msi) {
-                                nv_info(pmc, "MSI interrupts enabled\n");
-                                nv_wr08(device, 0x00088068, 0xff);
-                        }
+                        nv_info(pmc, "MSI interrupts enabled\n");
+                        oclass->msi_rearm(pmc);
                 }
-                break;
+        } else {
+                pmc->use_msi = false;
         }
 
         ret = request_irq(device->pdev->irq, nouveau_mc_intr,
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nv04.c
index 64aa4edb0d9d..2d787e4dfefa 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/mc/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nv04.c
@@ -22,17 +22,14 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/mc.h>
-
-struct nv04_mc_priv {
-        struct nouveau_mc base;
-};
+#include "nv04.h"
 
 const struct nouveau_mc_intr
 nv04_mc_intr[] = {
         { 0x00000001, NVDEV_ENGINE_MPEG },      /* NV17- MPEG/ME */
         { 0x00000100, NVDEV_ENGINE_FIFO },
         { 0x00001000, NVDEV_ENGINE_GR },
+        { 0x00010000, NVDEV_ENGINE_DISP },
         { 0x00020000, NVDEV_ENGINE_VP },        /* NV40- */
         { 0x00100000, NVDEV_SUBDEV_TIMER },
         { 0x01000000, NVDEV_ENGINE_DISP },      /* NV04- PCRTC0 */
@@ -42,7 +39,18 @@ nv04_mc_intr[] = {
         {}
 };
 
-static int
+int
+nv04_mc_init(struct nouveau_object *object)
+{
+        struct nv04_mc_priv *priv = (void *)object;
+
+        nv_wr32(priv, 0x000200, 0xffffffff); /* everything enabled */
+        nv_wr32(priv, 0x001850, 0x00000001); /* disable rom access */
+
+        return nouveau_mc_init(&priv->base);
+}
+
+int
 nv04_mc_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
              struct nouveau_oclass *oclass, void *data, u32 size,
              struct nouveau_object **pobject)
@@ -50,7 +58,7 @@ nv04_mc_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nv04_mc_priv *priv;
         int ret;
 
-        ret = nouveau_mc_create(parent, engine, oclass, nv04_mc_intr, &priv);
+        ret = nouveau_mc_create(parent, engine, oclass, &priv);
         *pobject = nv_object(priv);
         if (ret)
                 return ret;
@@ -58,24 +66,14 @@ nv04_mc_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-int
-nv04_mc_init(struct nouveau_object *object)
-{
-        struct nv04_mc_priv *priv = (void *)object;
-
-        nv_wr32(priv, 0x000200, 0xffffffff); /* everything enabled */
-        nv_wr32(priv, 0x001850, 0x00000001); /* disable rom access */
-
-        return nouveau_mc_init(&priv->base);
-}
-
-struct nouveau_oclass
-nv04_mc_oclass = {
-        .handle = NV_SUBDEV(MC, 0x04),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nv04_mc_oclass = &(struct nouveau_mc_oclass) {
+        .base.handle = NV_SUBDEV(MC, 0x04),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv04_mc_ctor,
                 .dtor = _nouveau_mc_dtor,
                 .init = nv04_mc_init,
                 .fini = _nouveau_mc_fini,
         },
-};
+        .intr = nv04_mc_intr,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nv04.h b/drivers/gpu/drm/nouveau/core/subdev/mc/nv04.h
new file mode 100644
index 000000000000..b0d5c31606c1
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nv04.h
@@ -0,0 +1,21 @@
+#ifndef __NVKM_MC_NV04_H__
+#define __NVKM_MC_NV04_H__
+
+#include <subdev/mc.h>
+
+struct nv04_mc_priv {
+        struct nouveau_mc base;
+};
+
+int  nv04_mc_ctor(struct nouveau_object *, struct nouveau_object *,
+                  struct nouveau_oclass *, void *, u32,
+                  struct nouveau_object **);
+
+extern const struct nouveau_mc_intr nv04_mc_intr[];
+int  nv04_mc_init(struct nouveau_object *);
+void nv40_mc_msi_rearm(struct nouveau_mc *);
+int  nv50_mc_init(struct nouveau_object *);
+extern const struct nouveau_mc_intr nv50_mc_intr[];
+extern const struct nouveau_mc_intr nvc0_mc_intr[];
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nv40.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nv40.c
new file mode 100644
index 000000000000..5b1faecfed2d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nv40.c
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv04.h"
+
+void
+nv40_mc_msi_rearm(struct nouveau_mc *pmc)
+{
+        struct nv04_mc_priv *priv = (void *)pmc;
+        nv_wr08(priv, 0x088068, 0xff);
+}
+
+struct nouveau_oclass *
+nv40_mc_oclass = &(struct nouveau_mc_oclass) {
+        .base.handle = NV_SUBDEV(MC, 0x40),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_mc_ctor,
+                .dtor = _nouveau_mc_dtor,
+                .init = nv04_mc_init,
+                .fini = _nouveau_mc_fini,
+        },
+        .intr = nv04_mc_intr,
+        .msi_rearm = nv40_mc_msi_rearm,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nv44.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nv44.c
index d9891782bf28..3bfee5c6c4f2 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/mc/nv44.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nv44.c
@@ -22,32 +22,12 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/mc.h>
-
-struct nv44_mc_priv {
-        struct nouveau_mc base;
-};
-
-static int
-nv44_mc_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv44_mc_priv *priv;
-        int ret;
-
-        ret = nouveau_mc_create(parent, engine, oclass, nv04_mc_intr, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        return 0;
-}
+#include "nv04.h"
 
 static int
 nv44_mc_init(struct nouveau_object *object)
 {
-        struct nv44_mc_priv *priv = (void *)object;
+        struct nv04_mc_priv *priv = (void *)object;
         u32 tmp = nv_rd32(priv, 0x10020c);
 
         nv_wr32(priv, 0x000200, 0xffffffff); /* everything enabled */
@@ -60,13 +40,15 @@ nv44_mc_init(struct nouveau_object *object)
         return nouveau_mc_init(&priv->base);
 }
 
-struct nouveau_oclass
-nv44_mc_oclass = {
-        .handle = NV_SUBDEV(MC, 0x44),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv44_mc_ctor,
+struct nouveau_oclass *
+nv44_mc_oclass = &(struct nouveau_mc_oclass) {
+        .base.handle = NV_SUBDEV(MC, 0x44),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_mc_ctor,
                 .dtor = _nouveau_mc_dtor,
                 .init = nv44_mc_init,
                 .fini = _nouveau_mc_fini,
         },
-};
+        .intr = nv04_mc_intr,
+        .msi_rearm = nv40_mc_msi_rearm,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nv50.c
index 2b1afe225db8..e8822a934c48 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/mc/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nv50.c
@@ -22,13 +22,9 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/mc.h>
+#include "nv04.h"
 
-struct nv50_mc_priv {
-        struct nouveau_mc base;
-};
-
-static const struct nouveau_mc_intr
+const struct nouveau_mc_intr
 nv50_mc_intr[] = {
         { 0x00000001, NVDEV_ENGINE_MPEG },
         { 0x00000100, NVDEV_ENGINE_FIFO },
@@ -45,37 +41,30 @@ nv50_mc_intr[] = {
         {},
 };
 
-static int
-nv50_mc_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
+static void
+nv50_mc_msi_rearm(struct nouveau_mc *pmc)
 {
-        struct nv50_mc_priv *priv;
-        int ret;
-
-        ret = nouveau_mc_create(parent, engine, oclass, nv50_mc_intr, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        return 0;
+        struct nouveau_device *device = nv_device(pmc);
+        pci_write_config_byte(device->pdev, 0x68, 0xff);
 }
 
 int
 nv50_mc_init(struct nouveau_object *object)
 {
-        struct nv50_mc_priv *priv = (void *)object;
+        struct nv04_mc_priv *priv = (void *)object;
         nv_wr32(priv, 0x000200, 0xffffffff); /* everything on */
         return nouveau_mc_init(&priv->base);
 }
 
-struct nouveau_oclass
-nv50_mc_oclass = {
-        .handle = NV_SUBDEV(MC, 0x50),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv50_mc_ctor,
+struct nouveau_oclass *
+nv50_mc_oclass = &(struct nouveau_mc_oclass) {
+        .base.handle = NV_SUBDEV(MC, 0x50),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_mc_ctor,
                 .dtor = _nouveau_mc_dtor,
                 .init = nv50_mc_init,
                 .fini = _nouveau_mc_fini,
         },
-};
+        .intr = nv50_mc_intr,
+        .msi_rearm = nv50_mc_msi_rearm,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nv94.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nv94.c
new file mode 100644
index 000000000000..5f4541105e73
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nv94.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv04.h"
+
+struct nouveau_oclass *
+nv94_mc_oclass = &(struct nouveau_mc_oclass) {
+        .base.handle = NV_SUBDEV(MC, 0x94),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_mc_ctor,
+                .dtor = _nouveau_mc_dtor,
+                .init = nv50_mc_init,
+                .fini = _nouveau_mc_fini,
+        },
+        .intr = nv50_mc_intr,
+        .msi_rearm = nv40_mc_msi_rearm,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nv98.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nv98.c
index 06710419a59b..f8a6f18e2d34 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/mc/nv98.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nv98.c
@@ -22,11 +22,7 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/mc.h>
-
-struct nv98_mc_priv {
-        struct nouveau_mc base;
-};
+#include "nv04.h"
 
 static const struct nouveau_mc_intr
 nv98_mc_intr[] = {
@@ -36,6 +32,7 @@ nv98_mc_intr[] = {
         { 0x00004000, NVDEV_ENGINE_CRYPT },     /* NV84:NVA3 */
         { 0x00008000, NVDEV_ENGINE_BSP },
         { 0x00020000, NVDEV_ENGINE_VP },
+        { 0x00040000, NVDEV_SUBDEV_PWR },       /* NVA3:NVC0 */
         { 0x00080000, NVDEV_SUBDEV_THERM },     /* NVA3:NVC0 */
         { 0x00100000, NVDEV_SUBDEV_TIMER },
         { 0x00200000, NVDEV_SUBDEV_GPIO },
@@ -47,29 +44,15 @@ nv98_mc_intr[] = {
         {},
 };
 
-static int
-nv98_mc_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
-{
-        struct nv98_mc_priv *priv;
-        int ret;
-
-        ret = nouveau_mc_create(parent, engine, oclass, nv98_mc_intr, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        return 0;
-}
-
-struct nouveau_oclass
-nv98_mc_oclass = {
-        .handle = NV_SUBDEV(MC, 0x98),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nv98_mc_ctor,
+struct nouveau_oclass *
+nv98_mc_oclass = &(struct nouveau_mc_oclass) {
+        .base.handle = NV_SUBDEV(MC, 0x98),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_mc_ctor,
                 .dtor = _nouveau_mc_dtor,
                 .init = nv50_mc_init,
                 .fini = _nouveau_mc_fini,
         },
-};
+        .intr = nv98_mc_intr,
+        .msi_rearm = nv40_mc_msi_rearm,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nvc0.c
index 104175c5a2dd..c02b4763a2d5 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/mc/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nvc0.c
@@ -22,13 +22,9 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/mc.h>
+#include "nv04.h"
 
-struct nvc0_mc_priv {
-        struct nouveau_mc base;
-};
-
-static const struct nouveau_mc_intr
+const struct nouveau_mc_intr
 nvc0_mc_intr[] = {
         { 0x00000001, NVDEV_ENGINE_PPP },
         { 0x00000020, NVDEV_ENGINE_COPY0 },
@@ -41,6 +37,7 @@ nvc0_mc_intr[] = {
         { 0x00020000, NVDEV_ENGINE_VP },
         { 0x00100000, NVDEV_SUBDEV_TIMER },
         { 0x00200000, NVDEV_SUBDEV_GPIO },
+        { 0x01000000, NVDEV_SUBDEV_PWR },
         { 0x02000000, NVDEV_SUBDEV_LTCG },
         { 0x04000000, NVDEV_ENGINE_DISP },
         { 0x10000000, NVDEV_SUBDEV_BUS },
@@ -49,29 +46,22 @@ nvc0_mc_intr[] = {
         {},
 };
 
-static int
-nvc0_mc_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-             struct nouveau_oclass *oclass, void *data, u32 size,
-             struct nouveau_object **pobject)
+static void
+nvc0_mc_msi_rearm(struct nouveau_mc *pmc)
 {
-        struct nvc0_mc_priv *priv;
-        int ret;
-
-        ret = nouveau_mc_create(parent, engine, oclass, nvc0_mc_intr, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        return 0;
+        struct nv04_mc_priv *priv = (void *)pmc;
+        nv_wr32(priv, 0x088704, 0x00000000);
 }
 
-struct nouveau_oclass
-nvc0_mc_oclass = {
-        .handle = NV_SUBDEV(MC, 0xc0),
-        .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nvc0_mc_ctor,
+struct nouveau_oclass *
+nvc0_mc_oclass = &(struct nouveau_mc_oclass) {
+        .base.handle = NV_SUBDEV(MC, 0xc0),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_mc_ctor,
                 .dtor = _nouveau_mc_dtor,
                 .init = nv50_mc_init,
                 .fini = _nouveau_mc_fini,
         },
-};
+        .intr = nvc0_mc_intr,
+        .msi_rearm = nvc0_mc_msi_rearm,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/nvc3.c b/drivers/gpu/drm/nouveau/core/subdev/mc/nvc3.c
new file mode 100644
index 000000000000..837e545aeb9f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/nvc3.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv04.h"
+
+struct nouveau_oclass *
+nvc3_mc_oclass = &(struct nouveau_mc_oclass) {
+        .base.handle = NV_SUBDEV(MC, 0xc3),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_mc_ctor,
+                .dtor = _nouveau_mc_dtor,
+                .init = nv50_mc_init,
+                .fini = _nouveau_mc_fini,
+        },
+        .intr = nvc0_mc_intr,
+        .msi_rearm = nv40_mc_msi_rearm,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/base.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/base.c
new file mode 100644
index 000000000000..9908f1f05a00
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/base.c
@@ -0,0 +1,247 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/pwr.h>
+#include <subdev/timer.h>
+
+static int
+nouveau_pwr_send(struct nouveau_pwr *ppwr, u32 reply[2],
+                 u32 process, u32 message, u32 data0, u32 data1)
+{
+        struct nouveau_subdev *subdev = nv_subdev(ppwr);
+        u32 addr;
+
+        /* we currently only support a single process at a time waiting
+         * on a synchronous reply, take the PPWR mutex and tell the
+         * receive handler what we're waiting for
+         */
+        if (reply) {
+                mutex_lock(&subdev->mutex);
+                ppwr->recv.message = message;
+                ppwr->recv.process = process;
+        }
+
+        /* wait for a free slot in the fifo */
+        addr  = nv_rd32(ppwr, 0x10a4a0);
+        if (!nv_wait_ne(ppwr, 0x10a4b0, 0xffffffff, addr ^ 8))
+                return -EBUSY;
+
+        /* acquire data segment access */
+        do {
+                nv_wr32(ppwr, 0x10a580, 0x00000001);
+        } while (nv_rd32(ppwr, 0x10a580) != 0x00000001);
+
+        /* write the packet */
+        nv_wr32(ppwr, 0x10a1c0, 0x01000000 | (((addr & 0x07) << 4) +
+                                ppwr->send.base));
+        nv_wr32(ppwr, 0x10a1c4, process);
+        nv_wr32(ppwr, 0x10a1c4, message);
+        nv_wr32(ppwr, 0x10a1c4, data0);
+        nv_wr32(ppwr, 0x10a1c4, data1);
+        nv_wr32(ppwr, 0x10a4a0, (addr + 1) & 0x0f);
+
+        /* release data segment access */
+        nv_wr32(ppwr, 0x10a580, 0x00000000);
+
+        /* wait for reply, if requested */
+        if (reply) {
+                wait_event(ppwr->recv.wait, (ppwr->recv.process == 0));
+                reply[0] = ppwr->recv.data[0];
+                reply[1] = ppwr->recv.data[1];
+                mutex_unlock(&subdev->mutex);
+        }
+
+        return 0;
+}
+
+static void
+nouveau_pwr_recv(struct work_struct *work)
+{
+        struct nouveau_pwr *ppwr =
+                container_of(work, struct nouveau_pwr, recv.work);
+        u32 process, message, data0, data1;
+
+        /* nothing to do if GET == PUT */
+        u32 addr =  nv_rd32(ppwr, 0x10a4cc);
+        if (addr == nv_rd32(ppwr, 0x10a4c8))
+                return;
+
+        /* acquire data segment access */
+        do {
+                nv_wr32(ppwr, 0x10a580, 0x00000002);
+        } while (nv_rd32(ppwr, 0x10a580) != 0x00000002);
+
+        /* read the packet */
+        nv_wr32(ppwr, 0x10a1c0, 0x02000000 | (((addr & 0x07) << 4) +
+                                ppwr->recv.base));
+        process = nv_rd32(ppwr, 0x10a1c4);
+        message = nv_rd32(ppwr, 0x10a1c4);
+        data0   = nv_rd32(ppwr, 0x10a1c4);
+        data1   = nv_rd32(ppwr, 0x10a1c4);
+        nv_wr32(ppwr, 0x10a4cc, (addr + 1) & 0x0f);
+
+        /* release data segment access */
+        nv_wr32(ppwr, 0x10a580, 0x00000000);
+
+        /* wake process if it's waiting on a synchronous reply */
+        if (ppwr->recv.process) {
+                if (process == ppwr->recv.process &&
+                    message == ppwr->recv.message) {
+                        ppwr->recv.data[0] = data0;
+                        ppwr->recv.data[1] = data1;
+                        ppwr->recv.process = 0;
+                        wake_up(&ppwr->recv.wait);
+                        return;
+                }
+        }
+
+        /* right now there's no other expected responses from the engine,
+         * so assume that any unexpected message is an error.
+         */
+        nv_warn(ppwr, "%c%c%c%c 0x%08x 0x%08x 0x%08x 0x%08x\n",
+                (char)((process & 0x000000ff) >>  0),
+                (char)((process & 0x0000ff00) >>  8),
+                (char)((process & 0x00ff0000) >> 16),
+                (char)((process & 0xff000000) >> 24),
+                process, message, data0, data1);
+}
+
+static void
+nouveau_pwr_intr(struct nouveau_subdev *subdev)
+{
+        struct nouveau_pwr *ppwr = (void *)subdev;
+        u32 disp = nv_rd32(ppwr, 0x10a01c);
+        u32 intr = nv_rd32(ppwr, 0x10a008) & disp & ~(disp >> 16);
+
+        if (intr & 0x00000020) {
+                u32 stat = nv_rd32(ppwr, 0x10a16c);
+                if (stat & 0x80000000) {
+                        nv_error(ppwr, "UAS fault at 0x%06x addr 0x%08x\n",
+                                 stat & 0x00ffffff, nv_rd32(ppwr, 0x10a168));
+                        nv_wr32(ppwr, 0x10a16c, 0x00000000);
+                        intr &= ~0x00000020;
+                }
+        }
+
+        if (intr & 0x00000040) {
+                schedule_work(&ppwr->recv.work);
+                nv_wr32(ppwr, 0x10a004, 0x00000040);
+                intr &= ~0x00000040;
+        }
+
+        if (intr & 0x00000080) {
+                nv_info(ppwr, "wr32 0x%06x 0x%08x\n", nv_rd32(ppwr, 0x10a7a0),
+                                                      nv_rd32(ppwr, 0x10a7a4));
+                nv_wr32(ppwr, 0x10a004, 0x00000080);
+                intr &= ~0x00000080;
+        }
+
+        if (intr) {
+                nv_error(ppwr, "intr 0x%08x\n", intr);
+                nv_wr32(ppwr, 0x10a004, intr);
+        }
+}
+
+int
+_nouveau_pwr_fini(struct nouveau_object *object, bool suspend)
+{
+        struct nouveau_pwr *ppwr = (void *)object;
+
+        nv_wr32(ppwr, 0x10a014, 0x00000060);
+        flush_work(&ppwr->recv.work);
+
+        return nouveau_subdev_fini(&ppwr->base, suspend);
+}
+
+int
+_nouveau_pwr_init(struct nouveau_object *object)
+{
+        struct nouveau_pwr *ppwr = (void *)object;
+        int ret, i;
+
+        ret = nouveau_subdev_init(&ppwr->base);
+        if (ret)
+                return ret;
+
+        nv_subdev(ppwr)->intr = nouveau_pwr_intr;
+        ppwr->message = nouveau_pwr_send;
+
+        /* prevent previous ucode from running, wait for idle, reset */
+        nv_wr32(ppwr, 0x10a014, 0x0000ffff); /* INTR_EN_CLR = ALL */
+        nv_wait(ppwr, 0x10a04c, 0xffffffff, 0x00000000);
+        nv_mask(ppwr, 0x000200, 0x00002000, 0x00000000);
+        nv_mask(ppwr, 0x000200, 0x00002000, 0x00002000);
+
+        /* upload data segment */
+        nv_wr32(ppwr, 0x10a1c0, 0x01000000);
+        for (i = 0; i < ppwr->data.size / 4; i++)
+                nv_wr32(ppwr, 0x10a1c4, ppwr->data.data[i]);
+
+        /* upload code segment */
+        nv_wr32(ppwr, 0x10a180, 0x01000000);
+        for (i = 0; i < ppwr->code.size / 4; i++) {
+                if ((i & 0x3f) == 0)
+                        nv_wr32(ppwr, 0x10a188, i >> 6);
+                nv_wr32(ppwr, 0x10a184, ppwr->code.data[i]);
+        }
+
+        /* start it running */
+        nv_wr32(ppwr, 0x10a10c, 0x00000000);
+        nv_wr32(ppwr, 0x10a104, 0x00000000);
+        nv_wr32(ppwr, 0x10a100, 0x00000002);
+
+        /* wait for valid host->pwr ring configuration */
+        if (!nv_wait_ne(ppwr, 0x10a4d0, 0xffffffff, 0x00000000))
+                return -EBUSY;
+        ppwr->send.base = nv_rd32(ppwr, 0x10a4d0) & 0x0000ffff;
+        ppwr->send.size = nv_rd32(ppwr, 0x10a4d0) >> 16;
+
+        /* wait for valid pwr->host ring configuration */
+        if (!nv_wait_ne(ppwr, 0x10a4dc, 0xffffffff, 0x00000000))
+                return -EBUSY;
+        ppwr->recv.base = nv_rd32(ppwr, 0x10a4dc) & 0x0000ffff;
+        ppwr->recv.size = nv_rd32(ppwr, 0x10a4dc) >> 16;
+
+        nv_wr32(ppwr, 0x10a010, 0x000000e0);
+        return 0;
+}
+
+int
+nouveau_pwr_create_(struct nouveau_object *parent,
+                    struct nouveau_object *engine,
+                    struct nouveau_oclass *oclass, int length, void **pobject)
+{
+        struct nouveau_pwr *ppwr;
+        int ret;
+
+        ret = nouveau_subdev_create_(parent, engine, oclass, 0, "PPWR",
+                                     "pwr", length, pobject);
+        ppwr = *pobject;
+        if (ret)
+                return ret;
+
+        INIT_WORK(&ppwr->recv.work, nouveau_pwr_recv);
+        init_waitqueue_head(&ppwr->recv.wait);
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/host.fuc b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/host.fuc
new file mode 100644
index 000000000000..2284ecb1c9b8
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/host.fuc
@@ -0,0 +1,151 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#ifdef INCLUDE_PROC
+process(PROC_HOST, #host_init, #host_recv)
+#endif
+
+/******************************************************************************
+ * HOST data segment
+ *****************************************************************************/
+#ifdef INCLUDE_DATA
+// HOST (R)FIFO packet format
+.equ #fifo_process 0x00
+.equ #fifo_message 0x04
+.equ #fifo_data0   0x08
+.equ #fifo_data1   0x0c
+
+// HOST HOST->PWR queue description
+.equ #fifo_qlen 4 // log2(size of queue entry in bytes)
+.equ #fifo_qnum 3 // log2(max number of entries in queue)
+.equ #fifo_qmaskb (1 << #fifo_qnum) // max number of entries in queue
+.equ #fifo_qmaskp (#fifo_qmaskb - 1)
+.equ #fifo_qmaskf ((#fifo_qmaskb << 1) - 1)
+.equ #fifo_qsize  (1 << (#fifo_qlen + #fifo_qnum))
+fifo_queue: .skip 128 // #fifo_qsize
+
+// HOST PWR->HOST queue description
+.equ #rfifo_qlen 4 // log2(size of queue entry in bytes)
+.equ #rfifo_qnum 3 // log2(max number of entries in queue)
+.equ #rfifo_qmaskb (1 << #rfifo_qnum) // max number of entries in queue
+.equ #rfifo_qmaskp (#rfifo_qmaskb - 1)
+.equ #rfifo_qmaskf ((#rfifo_qmaskb << 1) - 1)
+.equ #rfifo_qsize  (1 << (#rfifo_qlen + #rfifo_qnum))
+rfifo_queue: .skip 128 // #rfifo_qsize
+#endif
+
+/******************************************************************************
+ * HOST code segment
+ *****************************************************************************/
+#ifdef INCLUDE_CODE
+// HOST->PWR comms - dequeue message(s) for process(es) from FIFO
+//
+// $r15 - current (host)
+// $r0  - zero
+host_send:
+        nv_iord($r1, NV_PPWR_FIFO_GET(0))
+        nv_iord($r2, NV_PPWR_FIFO_PUT(0))
+        cmp b32 $r1 $r2
+        bra e #host_send_done
+                // calculate address of message
+                and $r14 $r1 #fifo_qmaskp
+                shl b32 $r14 $r14 #fifo_qlen
+                add b32 $r14 #fifo_queue
+
+                // read message data, and pass to appropriate process
+                ld b32 $r11 D[$r14 + #fifo_data1]
+                ld b32 $r12 D[$r14 + #fifo_data0]
+                ld b32 $r13 D[$r14 + #fifo_message]
+                ld b32 $r14 D[$r14 + #fifo_process]
+                call(send)
+
+                // increment GET
+                add b32 $r1 0x1
+                and $r14 $r1 #fifo_qmaskf
+                nv_iowr(NV_PPWR_FIFO_GET(0), $r1)
+                bra #host_send
+        host_send_done:
+        ret
+
+// PWR->HOST comms - enqueue message for HOST to RFIFO
+//
+// $r15 - current (host)
+// $r14 - process
+// $r13 - message
+// $r12 - message data 0
+// $r11 - message data 1
+// $r0  - zero
+host_recv:
+        // message from intr handler == HOST->PWR comms pending
+        mov $r1 (PROC_KERN & 0x0000ffff)
+        sethi $r1 (PROC_KERN & 0xffff0000)
+        cmp b32 $r14 $r1
+        bra e #host_send
+
+        // wait for space in RFIFO
+        host_recv_wait:
+        nv_iord($r1, NV_PPWR_RFIFO_GET)
+        nv_iord($r2, NV_PPWR_RFIFO_PUT)
+        xor $r1 #rfifo_qmaskb
+        cmp b32 $r1 $r2
+        bra e #host_recv_wait
+
+        and $r3 $r2 #rfifo_qmaskp
+        shl b32 $r3 #rfifo_qlen
+        add b32 $r3 #rfifo_queue
+
+        // enqueue message
+        st b32 D[$r3 + #fifo_data1] $r11
+        st b32 D[$r3 + #fifo_data0] $r12
+        st b32 D[$r3 + #fifo_message] $r13
+        st b32 D[$r3 + #fifo_process] $r14
+
+        add b32 $r2 0x1
+        and $r2 #rfifo_qmaskf
+        nv_iowr(NV_PPWR_RFIFO_PUT, $r2)
+
+        // notify host of pending message
+        mov $r2 NV_PPWR_INTR_TRIGGER_USER0
+        nv_iowr(NV_PPWR_INTR_TRIGGER, $r2)
+        ret
+
+// $r15 - current (host)
+// $r0  - zero
+host_init:
+        // store each fifo's base/size in H2D/D2H scratch regs
+        mov $r1 #fifo_qsize
+        shl b32 $r1 16
+        or $r1 #fifo_queue
+        nv_iowr(NV_PPWR_H2D, $r1);
+
+        mov $r1 #rfifo_qsize
+        shl b32 $r1 16
+        or $r1 #rfifo_queue
+        nv_iowr(NV_PPWR_D2H, $r1);
+
+        // enable fifo subintr for first fifo
+        mov $r1 1
+        nv_iowr(NV_PPWR_FIFO_INTR_EN, $r1)
+        ret
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/idle.fuc b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/idle.fuc
new file mode 100644
index 000000000000..98f1c3738b42
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/idle.fuc
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#ifdef INCLUDE_PROC
+process(PROC_IDLE, #idle, #idle_recv)
+#endif
+
+/******************************************************************************
+ * IDLE data segment
+ *****************************************************************************/
+#ifdef INCLUDE_DATA
+#endif
+
+/******************************************************************************
+ * IDLE code segment
+ *****************************************************************************/
+#ifdef INCLUDE_CODE
+// description
+//
+// $r15 - current (idle)
+// $r14 - message
+// $r0  - zero
+idle_recv:
+        ret
+
+// description
+//
+// $r15 - current (idle)
+// $r0  - zero
+idle:
+        // set our "no interrupt has occurred during our execution" flag
+        bset $flags $p0
+
+        // count IDLE invocations for debugging purposes
+        nv_iord($r1, NV_PPWR_DSCRATCH(1))
+        add b32 $r1 1
+        nv_iowr(NV_PPWR_DSCRATCH(1), $r1)
+
+        // keep looping while there's pending messages for any process
+        idle_loop:
+        mov $r1 #proc_list_head
+        bclr $flags $p2
+        idle_proc:
+                // process the process' messages until there's none left
+                idle_proc_exec:
+                        push $r1
+                        mov b32 $r14 $r1
+                        call(recv)
+                        pop $r1
+                        bra not $p1 #idle_proc_next
+                        bset $flags $p2
+                        bra #idle_proc_exec
+                // next process!
+                idle_proc_next:
+                add b32 $r1 #proc_size
+                cmp b32 $r1 $r15
+                bra ne #idle_proc
+        bra $p2 #idle_loop
+
+        // sleep if no interrupts have occurred
+        sleep $p0
+        bra #idle
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/kernel.fuc b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/kernel.fuc
new file mode 100644
index 000000000000..0a7b05fa5c11
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/kernel.fuc
@@ -0,0 +1,452 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+/******************************************************************************
+ * kernel data segment
+ *****************************************************************************/
+#ifdef INCLUDE_PROC
+proc_kern:
+process(PROC_KERN, 0, 0)
+proc_list_head:
+#endif
+
+#ifdef INCLUDE_DATA
+proc_list_tail:
+time_prev: .b32 0
+time_next: .b32 0
+#endif
+
+/******************************************************************************
+ * kernel code segment
+ *****************************************************************************/
+#ifdef INCLUDE_CODE
+        bra #init
+
+// read nv register
+//
+// $r15 - current
+// $r14 - addr
+// $r13 - data (return)
+// $r0  - zero
+rd32:
+        nv_iowr(NV_PPWR_MMIO_ADDR, $r14)
+        mov $r14 NV_PPWR_MMIO_CTRL_OP_RD
+        sethi $r14 NV_PPWR_MMIO_CTRL_TRIGGER
+        nv_iowr(NV_PPWR_MMIO_CTRL, $r14)
+        rd32_wait:
+                nv_iord($r14, NV_PPWR_MMIO_CTRL)
+                and $r14 NV_PPWR_MMIO_CTRL_STATUS
+                bra nz #rd32_wait
+        nv_iord($r13, NV_PPWR_MMIO_DATA)
+        ret
+
+// write nv register
+//
+// $r15 - current
+// $r14 - addr
+// $r13 - data
+// $r0  - zero
+wr32:
+        nv_iowr(NV_PPWR_MMIO_ADDR, $r14)
+        nv_iowr(NV_PPWR_MMIO_DATA, $r13)
+        mov $r14 NV_PPWR_MMIO_CTRL_OP_WR
+        or $r14 NV_PPWR_MMIO_CTRL_MASK_B32_0
+        sethi $r14 NV_PPWR_MMIO_CTRL_TRIGGER
+
+#ifdef NVKM_FALCON_MMIO_TRAP
+        mov $r8 NV_PPWR_INTR_TRIGGER_USER1
+        nv_iowr(NV_PPWR_INTR_TRIGGER, $r8)
+        wr32_host:
+                nv_iord($r8, NV_PPWR_INTR)
+                and $r8 NV_PPWR_INTR_USER1
+                bra nz #wr32_host
+#endif
+
+        nv_iowr(NV_PPWR_MMIO_CTRL, $r14)
+        wr32_wait:
+                nv_iord($r14, NV_PPWR_MMIO_CTRL)
+                and $r14 NV_PPWR_MMIO_CTRL_STATUS
+                bra nz #wr32_wait
+        ret
+
+// busy-wait for a period of time
+//
+// $r15 - current
+// $r14 - ns
+// $r0  - zero
+nsec:
+        nv_iord($r8, NV_PPWR_TIMER_LOW)
+        nsec_loop:
+                nv_iord($r9, NV_PPWR_TIMER_LOW)
+                sub b32 $r9 $r8
+                cmp b32 $r9 $r14
+                bra l #nsec_loop
+        ret
+
+// busy-wait for a period of time
+//
+// $r15 - current
+// $r14 - addr
+// $r13 - mask
+// $r12 - data
+// $r11 - timeout (ns)
+// $r0  - zero
+wait:
+        nv_iord($r8, NV_PPWR_TIMER_LOW)
+        wait_loop:
+                nv_rd32($r10, $r14)
+                and $r10 $r13
+                cmp b32 $r10 $r12
+                bra e #wait_done
+                nv_iord($r9, NV_PPWR_TIMER_LOW)
+                sub b32 $r9 $r8
+                cmp b32 $r9 $r11
+                bra l #wait_loop
+        wait_done:
+        ret
+
+// $r15 - current (kern)
+// $r14 - process
+// $r8  - NV_PPWR_INTR
+intr_watchdog:
+        // read process' timer status, skip if not enabled
+        ld b32 $r9 D[$r14 + #proc_time]
+        cmp b32 $r9 0
+        bra z #intr_watchdog_next_proc
+
+        // subtract last timer's value from process' timer,
+        // if it's <= 0 then the timer has expired
+        ld b32 $r10 D[$r0 + #time_prev]
+        sub b32 $r9 $r10
+        bra g #intr_watchdog_next_time
+                mov $r13 KMSG_ALARM
+                call(send_proc)
+                clear b32 $r9
+                bra #intr_watchdog_next_proc
+
+        // otherwise, update the next timer's value if this
+        // process' timer is the soonest
+        intr_watchdog_next_time:
+                // ... or if there's no next timer yet
+                ld b32 $r10 D[$r0 + #time_next]
+                cmp b32 $r10 0
+                bra z #intr_watchdog_next_time_set
+
+                cmp b32 $r9 $r10
+                bra g #intr_watchdog_next_proc
+                intr_watchdog_next_time_set:
+                st b32 D[$r0 + #time_next] $r9
+
+        // update process' timer status, and advance
+        intr_watchdog_next_proc:
+        st b32 D[$r14 + #proc_time] $r9
+        add b32 $r14 #proc_size
+        cmp b32 $r14 #proc_list_tail
+        bra ne #intr_watchdog
+        ret
+
+intr:
+        push $r0
+        clear b32 $r0
+        push $r8
+        push $r9
+        push $r10
+        push $r11
+        push $r12
+        push $r13
+        push $r14
+        push $r15
+        mov $r15 #proc_kern
+        mov $r8 $flags
+        push $r8
+
+        nv_iord($r8, NV_PPWR_DSCRATCH(0))
+        add b32 $r8 1
+        nv_iowr(NV_PPWR_DSCRATCH(0), $r8)
+
+        nv_iord($r8, NV_PPWR_INTR)
+        and $r9 $r8 NV_PPWR_INTR_WATCHDOG
+        bra z #intr_skip_watchdog
+                st b32 D[$r0 + #time_next] $r0
+                mov $r14 #proc_list_head
+                call(intr_watchdog)
+                ld b32 $r9 D[$r0 + #time_next]
+                cmp b32 $r9 0
+                bra z #intr_skip_watchdog
+                        nv_iowr(NV_PPWR_WATCHDOG_TIME, $r9)
+                        st b32 D[$r0 + #time_prev] $r9
+
+        intr_skip_watchdog:
+        and $r9 $r8 NV_PPWR_INTR_SUBINTR
+        bra z #intr_skip_subintr
+                nv_iord($r9, NV_PPWR_SUBINTR)
+                and $r10 $r9 NV_PPWR_SUBINTR_FIFO
+                bra z #intr_subintr_skip_fifo
+                        nv_iord($r12, NV_PPWR_FIFO_INTR)
+                        push $r12
+                        mov $r14 (PROC_HOST & 0x0000ffff)
+                        sethi $r14 (PROC_HOST & 0xffff0000)
+                        mov $r13 KMSG_FIFO
+                        call(send)
+                        pop $r12
+                        nv_iowr(NV_PPWR_FIFO_INTR, $r12)
+                intr_subintr_skip_fifo:
+                nv_iowr(NV_PPWR_SUBINTR, $r9)
+
+        intr_skip_subintr:
+        and $r9 $r8 NV_PPWR_INTR_PAUSE
+        bra z #intr_skip_pause
+                and $r10 0xffbf
+
+        intr_skip_pause:
+        and $r9 $r8 NV_PPWR_INTR_USER0
+        bra z #intr_skip_user0
+                and $r10 0xffbf
+
+        intr_skip_user0:
+        nv_iowr(NV_PPWR_INTR_ACK, $r8)
+        pop $r8
+        mov $flags $r8
+        pop $r15
+        pop $r14
+        pop $r13
+        pop $r12
+        pop $r11
+        pop $r10
+        pop $r9
+        pop $r8
+        pop $r0
+        bclr $flags $p0
+        iret
+
+// request the current process be sent a message after a timeout expires
+//
+// $r15 - current
+// $r14 - ticks
+// $r0  - zero
+timer:
+        // interrupts off to prevent racing with timer isr
+        bclr $flags ie0
+
+        // if current process already has a timer set, bail
+        ld b32 $r8 D[$r15 + #proc_time]
+        cmp b32 $r8 0
+        bra g #timer_done
+        st b32 D[$r15 + #proc_time] $r14
+
+        // halt watchdog timer temporarily and check for a pending
+        // interrupt.  if there's one already pending, we can just
+        // bail since the timer isr will queue the next soonest
+        // right after it's done
+        nv_iowr(NV_PPWR_WATCHDOG_ENABLE, $r8)
+        nv_iord($r8, NV_PPWR_INTR)
+        and $r8 NV_PPWR_INTR_WATCHDOG
+        bra nz #timer_enable
+
+        // update the watchdog if this timer should expire first,
+        // or if there's no timeout already set
+        nv_iord($r8, NV_PPWR_WATCHDOG_TIME)
+        cmp b32 $r14 $r0
+        bra e #timer_reset
+        cmp b32 $r14 $r8
+        bra l #timer_done
+        timer_reset:
+        nv_iowr(NV_PPWR_WATCHDOG_TIME, $r14)
+        st b32 D[$r0 + #time_prev] $r14
+
+        // re-enable the watchdog timer
+        timer_enable:
+        mov $r8 1
+        nv_iowr(NV_PPWR_WATCHDOG_ENABLE, $r8)
+
+        // interrupts back on
+        timer_done:
+        bset $flags ie0
+        ret
+
+// send message to another process
+//
+// $r15 - current
+// $r14 - process
+// $r13 - message
+// $r12 - message data 0
+// $r11 - message data 1
+// $r0  - zero
+send_proc:
+        push $r8
+        push $r9
+        // check for space in queue
+        ld b32 $r8 D[$r14 + #proc_qget]
+        ld b32 $r9 D[$r14 + #proc_qput]
+        xor $r8 #proc_qmaskb
+        cmp b32 $r8 $r9
+        bra e #send_done
+
+        // enqueue message
+        and $r8 $r9 #proc_qmaskp
+        shl b32 $r8 $r8 #proc_qlen
+        add b32 $r8 #proc_queue
+        add b32 $r8 $r14
+
+        ld b32 $r10 D[$r15 + #proc_id]
+        st b32 D[$r8 + #msg_process] $r10
+        st b32 D[$r8 + #msg_message] $r13
+        st b32 D[$r8 + #msg_data0] $r12
+        st b32 D[$r8 + #msg_data1] $r11
+
+        // increment PUT
+        add b32 $r9 1
+        and $r9 #proc_qmaskf
+        st b32 D[$r14 + #proc_qput] $r9
+        bset $flags $p2
+        send_done:
+        pop $r9
+        pop $r8
+        ret
+
+// lookup process structure by its name
+//
+// $r15 - current
+// $r14 - process name
+// $r0  - zero
+//
+// $r14 - process
+// $p1  - success
+find:
+        push $r8
+        mov $r8 #proc_list_head
+        bset $flags $p1
+        find_loop:
+                ld b32 $r10 D[$r8 + #proc_id]
+                cmp b32 $r10 $r14
+                bra e #find_done
+                add b32 $r8 #proc_size
+                cmp b32 $r8 #proc_list_tail
+                bra ne #find_loop
+                bclr $flags $p1
+        find_done:
+        mov b32 $r14 $r8
+        pop $r8
+        ret
+
+// send message to another process
+//
+// $r15 - current
+// $r14 - process id
+// $r13 - message
+// $r12 - message data 0
+// $r11 - message data 1
+// $r0  - zero
+send:
+        call(find)
+        bra $p1 #send_proc
+        ret
+
+// process single message for a given process
+//
+// $r15 - current
+// $r14 - process
+// $r0  - zero
+recv:
+        ld b32 $r8 D[$r14 + #proc_qget]
+        ld b32 $r9 D[$r14 + #proc_qput]
+        bclr $flags $p1
+        cmp b32 $r8 $r9
+        bra e #recv_done
+                // dequeue message
+                and $r9 $r8 #proc_qmaskp
+                add b32 $r8 1
+                and $r8 #proc_qmaskf
+                st b32 D[$r14 + #proc_qget] $r8
+                ld b32 $r10 D[$r14 + #proc_recv]
+
+                push $r15
+                mov $r15 $flags
+                push $r15
+                mov b32 $r15 $r14
+
+                shl b32 $r9 $r9 #proc_qlen
+                add b32 $r14 $r9
+                add b32 $r14 #proc_queue
+                ld b32 $r11 D[$r14 + #msg_data1]
+                ld b32 $r12 D[$r14 + #msg_data0]
+                ld b32 $r13 D[$r14 + #msg_message]
+                ld b32 $r14 D[$r14 + #msg_process]
+
+                // process it
+                call $r10
+                pop $r15
+                mov $flags $r15
+                bset $flags $p1
+                pop $r15
+        recv_done:
+        ret
+
+init:
+        // setup stack
+        nv_iord($r1, NV_PPWR_CAPS)
+        extr $r1 $r1 9:17
+        shl b32 $r1 8
+        mov $sp $r1
+
+#ifdef NVKM_FALCON_MMIO_UAS
+        // somehow allows the magic "access mmio via D[]" stuff that's
+        // used by the nv_rd32/nv_wr32 macros to work
+        mov $r1 0x0010
+        sethi $r1 NV_PPWR_UAS_CONFIG_ENABLE
+        nv_iowrs(NV_PPWR_UAS_CONFIG, $r1)
+#endif
+
+        // route all interrupts except user0/1 and pause to fuc
+        mov $r1 0x00e0
+        sethi $r1 0x00000000
+        nv_iowr(NV_PPWR_INTR_ROUTE, $r1)
+
+        // enable watchdog and subintr intrs
+        mov $r1 NV_PPWR_INTR_EN_CLR_MASK
+        nv_iowr(NV_PPWR_INTR_EN_CLR, $r1)
+        mov $r1 NV_PPWR_INTR_EN_SET_WATCHDOG
+        or $r1 NV_PPWR_INTR_EN_SET_SUBINTR
+        nv_iowr(NV_PPWR_INTR_EN_SET, $r1)
+
+        // enable interrupts globally
+        mov $r1 #intr
+        sethi $r1 0x00000000
+        mov $iv0 $r1
+        bset $flags ie0
+
+        // enable watchdog timer
+        mov $r1 1
+        nv_iowr(NV_PPWR_WATCHDOG_ENABLE, $r1)
+
+        // bootstrap processes, idle process will be last, and not return
+        mov $r15 #proc_list_head
+        init_proc:
+                ld b32 $r1 D[$r15 + #proc_init]
+                cmp b32 $r1 0
+                bra z #init_proc
+                call $r1
+                add b32 $r15 #proc_size
+                bra #init_proc
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/macros.fuc b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/macros.fuc
new file mode 100644
index 000000000000..2a74ea907604
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/macros.fuc
@@ -0,0 +1,199 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#define GT215 0xa3
+#define GF100 0xc0
+#define GF119 0xd9
+#define GK208 0x108
+
+#include "os.h"
+
+// IO addresses
+#define NV_PPWR_INTR_TRIGGER                                             0x0000
+#define NV_PPWR_INTR_TRIGGER_USER1                                   0x00000080
+#define NV_PPWR_INTR_TRIGGER_USER0                                   0x00000040
+#define NV_PPWR_INTR_ACK                                                 0x0004
+#define NV_PPWR_INTR_ACK_SUBINTR                                     0x00000800
+#define NV_PPWR_INTR_ACK_WATCHDOG                                    0x00000002
+#define NV_PPWR_INTR                                                     0x0008
+#define NV_PPWR_INTR_SUBINTR                                         0x00000800
+#define NV_PPWR_INTR_USER1                                           0x00000080
+#define NV_PPWR_INTR_USER0                                           0x00000040
+#define NV_PPWR_INTR_PAUSE                                           0x00000020
+#define NV_PPWR_INTR_WATCHDOG                                        0x00000002
+#define NV_PPWR_INTR_EN_SET                                              0x0010
+#define NV_PPWR_INTR_EN_SET_SUBINTR                                  0x00000800
+#define NV_PPWR_INTR_EN_SET_WATCHDOG                                 0x00000002
+#define NV_PPWR_INTR_EN_CLR                                              0x0014
+#define NV_PPWR_INTR_EN_CLR_MASK                    /* fuck i hate envyas */ -1
+#define NV_PPWR_INTR_ROUTE                                               0x001c
+#define NV_PPWR_TIMER_LOW                                                0x002c
+#define NV_PPWR_WATCHDOG_TIME                                            0x0034
+#define NV_PPWR_WATCHDOG_ENABLE                                          0x0038
+#define NV_PPWR_CAPS                                                     0x0108
+#define NV_PPWR_UAS_CONFIG                                               0x0164
+#define NV_PPWR_UAS_CONFIG_ENABLE                                    0x00010000
+#if NVKM_PPWR_CHIPSET >= GK208
+#define NV_PPWR_DSCRATCH(i)                                   (4 * (i) + 0x0450)
+#endif
+#define NV_PPWR_FIFO_PUT(i)                                   (4 * (i) + 0x04a0)
+#define NV_PPWR_FIFO_GET(i)                                   (4 * (i) + 0x04b0)
+#define NV_PPWR_FIFO_INTR                                                0x04c0
+#define NV_PPWR_FIFO_INTR_EN                                             0x04c4
+#define NV_PPWR_RFIFO_PUT                                                0x04c8
+#define NV_PPWR_RFIFO_GET                                                0x04cc
+#define NV_PPWR_H2D                                                      0x04d0
+#define NV_PPWR_D2H                                                      0x04dc
+#if NVKM_PPWR_CHIPSET < GK208
+#define NV_PPWR_DSCRATCH(i)                                   (4 * (i) + 0x05d0)
+#endif
+#define NV_PPWR_SUBINTR                                                  0x0688
+#define NV_PPWR_SUBINTR_FIFO                                         0x00000002
+#define NV_PPWR_MMIO_ADDR                                                0x07a0
+#define NV_PPWR_MMIO_DATA                                                0x07a4
+#define NV_PPWR_MMIO_CTRL                                                0x07ac
+#define NV_PPWR_MMIO_CTRL_TRIGGER                                    0x00010000
+#define NV_PPWR_MMIO_CTRL_STATUS                                     0x00007000
+#define NV_PPWR_MMIO_CTRL_STATUS_IDLE                                0x00000000
+#define NV_PPWR_MMIO_CTRL_MASK                                       0x000000f0
+#define NV_PPWR_MMIO_CTRL_MASK_B32_0                                 0x000000f0
+#define NV_PPWR_MMIO_CTRL_OP                                         0x00000003
+#define NV_PPWR_MMIO_CTRL_OP_RD                                      0x00000001
+#define NV_PPWR_MMIO_CTRL_OP_WR                                      0x00000002
+#define NV_PPWR_OUTPUT                                                   0x07c0
+#define NV_PPWR_OUTPUT_FB_PAUSE                                      0x00000004
+#define NV_PPWR_OUTPUT_SET                                               0x07e0
+#define NV_PPWR_OUTPUT_SET_FB_PAUSE                                  0x00000004
+#define NV_PPWR_OUTPUT_CLR                                               0x07e4
+#define NV_PPWR_OUTPUT_CLR_FB_PAUSE                                  0x00000004
+
+// Inter-process message format
+.equ #msg_process 0x00 /* send() target, recv() sender */
+.equ #msg_message 0x04
+.equ #msg_data0   0x08
+.equ #msg_data1   0x0c
+
+// Kernel message IDs
+#define KMSG_FIFO  0x00000000
+#define KMSG_ALARM 0x00000001
+
+// Process message queue description
+.equ #proc_qlen 4 // log2(size of queue entry in bytes)
+.equ #proc_qnum 2 // log2(max number of entries in queue)
+.equ #proc_qmaskb (1 << #proc_qnum) // max number of entries in queue
+.equ #proc_qmaskp (#proc_qmaskb - 1)
+.equ #proc_qmaskf ((#proc_qmaskb << 1) - 1)
+.equ #proc_qsize  (1 << (#proc_qlen + #proc_qnum))
+
+// Process table entry
+.equ #proc_id    0x00
+.equ #proc_init  0x04
+.equ #proc_recv  0x08
+.equ #proc_time  0x0c
+.equ #proc_qput  0x10
+.equ #proc_qget  0x14
+.equ #proc_queue 0x18
+.equ #proc_size (0x18 + #proc_qsize)
+
+#define process(id,init,recv) /*
+*/      .b32 id /*
+*/      .b32 init /*
+*/      .b32 recv /*
+*/      .b32 0 /*
+*/      .b32 0 /*
+*/      .b32 0 /*
+*/      .skip 64
+
+#ifndef NVKM_FALCON_UNSHIFTED_IO
+#define nv_iord(reg,ior) /*
+*/      mov reg ior /*
+*/      shl b32 reg 6 /*
+*/      iord reg I[reg + 0x000]
+#else
+#define nv_iord(reg,ior) /*
+*/      mov reg ior /*
+*/      iord reg I[reg + 0x000]
+#endif
+
+#ifndef NVKM_FALCON_UNSHIFTED_IO
+#define nv_iowr(ior,reg) /*
+*/      mov $r0 ior /*
+*/      shl b32 $r0 6 /*
+*/      iowr I[$r0 + 0x000] reg /*
+*/      clear b32 $r0
+#else
+#define nv_iowr(ior,reg) /*
+*/      mov $r0 ior /*
+*/      iowr I[$r0 + 0x000] reg /*
+*/      clear b32 $r0
+#endif
+
+#ifndef NVKM_FALCON_UNSHIFTED_IO
+#define nv_iowrs(ior,reg) /*
+*/      mov $r0 ior /*
+*/      shl b32 $r0 6 /*
+*/      iowrs I[$r0 + 0x000] reg /*
+*/      clear b32 $r0
+#else
+#define nv_iowrs(ior,reg) /*
+*/      mov $r0 ior /*
+*/      iowrs I[$r0 + 0x000] reg /*
+*/      clear b32 $r0
+#endif
+
+#define hash #
+#define fn(a) a
+#ifndef NVKM_FALCON_PC24
+#define call(a) call fn(hash)a
+#else
+#define call(a) lcall fn(hash)a
+#endif
+
+#ifndef NVKM_FALCON_MMIO_UAS
+#define nv_rd32(reg,addr) /*
+*/      mov b32 $r14 addr /*
+*/      call(rd32) /*
+*/      mov b32 reg $r13
+#else
+#define nv_rd32(reg,addr) /*
+*/      sethi $r0 0x14000000 /*
+*/      or $r0 addr /*
+*/      ld b32 reg D[$r0] /*
+*/      clear b32 $r0
+#endif
+
+#if !defined(NVKM_FALCON_MMIO_UAS) || defined(NVKM_FALCON_MMIO_TRAP)
+#define nv_wr32(addr,reg) /*
+*/      push addr /*
+*/      push reg /*
+*/      pop $r13 /*
+*/      pop $r14 /*
+*/      call(wr32) /*
+#else
+#define nv_wr32(addr,reg) /*
+*/      sethi $r0 0x14000000 /*
+*/      or $r0 addr /*
+*/      st b32 D[$r0] reg /*
+*/      clear b32 $r0
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/memx.fuc b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/memx.fuc
new file mode 100644
index 000000000000..d43741eccb11
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/memx.fuc
@@ -0,0 +1,219 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#ifdef INCLUDE_PROC
+process(PROC_MEMX, #memx_init, #memx_recv)
+#endif
+
+/******************************************************************************
+ * MEMX data segment
+ *****************************************************************************/
+#ifdef INCLUDE_DATA
+.equ #memx_opcode 0
+.equ #memx_header 2
+.equ #memx_length 4
+.equ #memx_func   8
+
+#define handler(cmd,hdr,len,func) /*
+*/      .b16 MEMX_##cmd /*
+*/      .b16 hdr /*
+*/      .b16 len /*
+*/      .b16 0 /*
+*/      .b32 func
+
+memx_func_head:
+handler(ENTER , 0x0001, 0x0000, #memx_func_enter)
+memx_func_next:
+handler(LEAVE , 0x0000, 0x0000, #memx_func_leave)
+handler(WR32  , 0x0000, 0x0002, #memx_func_wr32)
+handler(WAIT  , 0x0004, 0x0000, #memx_func_wait)
+handler(DELAY , 0x0001, 0x0000, #memx_func_delay)
+memx_func_tail:
+
+.equ #memx_func_size #memx_func_next - #memx_func_head
+.equ #memx_func_num (#memx_func_tail - #memx_func_head) / #memx_func_size
+
+memx_data_head:
+.skip 0x0800
+memx_data_tail:
+#endif
+
+/******************************************************************************
+ * MEMX code segment
+ *****************************************************************************/
+#ifdef INCLUDE_CODE
+// description
+//
+// $r15 - current (memx)
+// $r4  - packet length
+//      +00: bitmask of heads to wait for vblank on
+// $r3  - opcode desciption
+// $r0  - zero
+memx_func_enter:
+        mov $r6 NV_PPWR_OUTPUT_SET_FB_PAUSE
+        nv_iowr(NV_PPWR_OUTPUT_SET, $r6)
+        memx_func_enter_wait:
+                nv_iord($r6, NV_PPWR_OUTPUT)
+                and $r6 NV_PPWR_OUTPUT_FB_PAUSE
+                bra z #memx_func_enter_wait
+        //XXX: TODO
+        ld b32 $r6 D[$r1 + 0x00]
+        add b32 $r1 0x04
+        ret
+
+// description
+//
+// $r15 - current (memx)
+// $r4  - packet length
+// $r3  - opcode desciption
+// $r0  - zero
+memx_func_leave:
+        mov $r6 NV_PPWR_OUTPUT_CLR_FB_PAUSE
+        nv_iowr(NV_PPWR_OUTPUT_CLR, $r6)
+        memx_func_leave_wait:
+                nv_iord($r6, NV_PPWR_OUTPUT)
+                and $r6 NV_PPWR_OUTPUT_FB_PAUSE
+                bra nz #memx_func_leave_wait
+        ret
+
+// description
+//
+// $r15 - current (memx)
+// $r4  - packet length
+//      +00*n: addr
+//      +04*n: data
+// $r3  - opcode desciption
+// $r0  - zero
+memx_func_wr32:
+        ld b32 $r6 D[$r1 + 0x00]
+        ld b32 $r5 D[$r1 + 0x04]
+        add b32 $r1 0x08
+        nv_wr32($r6, $r5)
+        sub b32 $r4 0x02
+        bra nz #memx_func_wr32
+        ret
+
+// description
+//
+// $r15 - current (memx)
+// $r4  - packet length
+//      +00: addr
+//      +04: mask
+//      +08: data
+//      +0c: timeout (ns)
+// $r3  - opcode desciption
+// $r0  - zero
+memx_func_wait:
+        nv_iord($r8, NV_PPWR_TIMER_LOW)
+        ld b32 $r14 D[$r1 + 0x00]
+        ld b32 $r13 D[$r1 + 0x04]
+        ld b32 $r12 D[$r1 + 0x08]
+        ld b32 $r11 D[$r1 + 0x0c]
+        add b32 $r1 0x10
+        call(wait)
+        ret
+
+// description
+//
+// $r15 - current (memx)
+// $r4  - packet length
+//      +00: time (ns)
+// $r3  - opcode desciption
+// $r0  - zero
+memx_func_delay:
+        ld b32 $r14 D[$r1 + 0x00]
+        add b32 $r1 0x04
+        call(nsec)
+        ret
+
+// description
+//
+// $r15 - current (memx)
+// $r14 - sender process name
+// $r13 - message (exec)
+// $r12 - head of script
+// $r11 - tail of script
+// $r0  - zero
+memx_exec:
+        push $r14
+        push $r13
+        mov b32 $r1 $r12
+        mov b32 $r2 $r11
+        memx_exec_next:
+                // fetch the packet header, and locate opcode info
+                ld b32 $r3 D[$r1]
+                add b32 $r1 4
+                shr b32 $r4 $r3 16
+                mulu $r3 #memx_func_size
+
+                // execute the opcode handler
+                ld b32 $r5 D[$r3 + #memx_func_head + #memx_func]
+                call $r5
+
+                // keep going, if we haven't reached the end
+                cmp b32 $r1 $r2
+                bra l #memx_exec_next
+
+        // send completion reply
+        pop $r13
+        pop $r14
+        call(send)
+        ret
+
+// description
+//
+// $r15 - current (memx)
+// $r14 - sender process name
+// $r13 - message
+// $r12 - data0
+// $r11 - data1
+// $r0  - zero
+memx_info:
+        mov $r12 #memx_data_head
+        mov $r11 #memx_data_tail - #memx_data_head
+        call(send)
+        ret
+
+// description
+//
+// $r15 - current (memx)
+// $r14 - sender process name
+// $r13 - message
+// $r12 - data0
+// $r11 - data1
+// $r0  - zero
+memx_recv:
+        cmp b32 $r13 MEMX_MSG_EXEC
+        bra e #memx_exec
+        cmp b32 $r13 MEMX_MSG_INFO
+        bra e #memx_info
+        ret
+
+// description
+//
+// $r15 - current (memx)
+// $r0  - zero
+memx_init:
+        ret
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc
new file mode 100644
index 000000000000..947be536daef
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#define NVKM_PPWR_CHIPSET GK208
+
+#define NVKM_FALCON_PC24
+#define NVKM_FALCON_UNSHIFTED_IO
+//#define NVKM_FALCON_MMIO_UAS
+//#define NVKM_FALCON_MMIO_TRAP
+
+#include "macros.fuc"
+
+.section #nv108_pwr_data
+#define INCLUDE_PROC
+#include "kernel.fuc"
+#include "host.fuc"
+#include "memx.fuc"
+#include "perf.fuc"
+#include "test.fuc"
+#include "idle.fuc"
+#undef INCLUDE_PROC
+
+#define INCLUDE_DATA
+#include "kernel.fuc"
+#include "host.fuc"
+#include "memx.fuc"
+#include "perf.fuc"
+#include "test.fuc"
+#include "idle.fuc"
+#undef INCLUDE_DATA
+.align 256
+
+.section #nv108_pwr_code
+#define INCLUDE_CODE
+#include "kernel.fuc"
+#include "host.fuc"
+#include "memx.fuc"
+#include "perf.fuc"
+#include "test.fuc"
+#include "idle.fuc"
+#undef INCLUDE_CODE
+.align 256
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h
new file mode 100644
index 000000000000..9342e2d7d3b7
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nv108.fuc.h
@@ -0,0 +1,1165 @@
+uint32_t nv108_pwr_data[] = {
+/* 0x0000: proc_kern */
+        0x52544e49,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0058: proc_list_head */
+        0x54534f48,
+        0x00000379,
+        0x0000032a,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x584d454d,
+        0x0000046f,
+        0x00000461,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x46524550,
+        0x00000473,
+        0x00000471,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x54534554,
+        0x00000494,
+        0x00000475,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x454c4449,
+        0x0000049f,
+        0x0000049d,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0210: proc_list_tail */
+/* 0x0210: time_prev */
+        0x00000000,
+/* 0x0214: time_next */
+        0x00000000,
+/* 0x0218: fifo_queue */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0298: rfifo_queue */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0318: memx_func_head */
+        0x00010000,
+        0x00000000,
+        0x000003a9,
+/* 0x0324: memx_func_next */
+        0x00000001,
+        0x00000000,
+        0x000003c7,
+        0x00000002,
+        0x00000002,
+        0x000003df,
+        0x00040003,
+        0x00000000,
+        0x00000407,
+        0x00010004,
+        0x00000000,
+        0x00000421,
+/* 0x0354: memx_func_tail */
+/* 0x0354: memx_data_head */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0b54: memx_data_tail */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
+
+uint32_t nv108_pwr_code[] = {
+        0x02910ef5,
+/* 0x0004: rd32 */
+        0xf607a040,
+        0x04bd000e,
+        0xe3f0010e,
+        0x07ac4001,
+        0xbd000ef6,
+/* 0x0019: rd32_wait */
+        0x07ac4e04,
+        0xf100eecf,
+        0xf47000e4,
+        0xa44df61b,
+        0x00ddcf07,
+/* 0x002e: wr32 */
+        0xa04000f8,
+        0x000ef607,
+        0xa44004bd,
+        0x000df607,
+        0x020e04bd,
+        0xf0f0e5f0,
+        0xac4001e3,
+        0x000ef607,
+/* 0x004e: wr32_wait */
+        0xac4e04bd,
+        0x00eecf07,
+        0x7000e4f1,
+        0xf8f61bf4,
+/* 0x005d: nsec */
+        0xcf2c0800,
+/* 0x0062: nsec_loop */
+        0x2c090088,
+        0xbb0099cf,
+        0x9ea60298,
+        0xf8f61ef4,
+/* 0x0071: wait */
+        0xcf2c0800,
+/* 0x0076: wait_loop */
+        0xeeb20088,
+        0x0000047e,
+        0xadfddab2,
+        0xf4aca604,
+        0x2c09100b,
+        0xbb0099cf,
+        0x9ba60298,
+/* 0x0093: wait_done */
+        0xf8e61ef4,
+/* 0x0095: intr_watchdog */
+        0x03e99800,
+        0xf40096b0,
+        0x0a98280b,
+        0x029abb84,
+        0x0d0e1cf4,
+        0x01de7e01,
+        0xf494bd00,
+/* 0x00b2: intr_watchdog_next_time */
+        0x0a98140e,
+        0x00a6b085,
+        0xa6080bf4,
+        0x061cf49a,
+/* 0x00c0: intr_watchdog_next_time_set */
+/* 0x00c3: intr_watchdog_next_proc */
+        0xb58509b5,
+        0xe0b603e9,
+        0x10e6b158,
+        0xc81bf402,
+/* 0x00d2: intr */
+        0x00f900f8,
+        0x80f904bd,
+        0xa0f990f9,
+        0xc0f9b0f9,
+        0xe0f9d0f9,
+        0x000ff0f9,
+        0xf90188fe,
+        0x04504880,
+        0xb60088cf,
+        0x50400180,
+        0x0008f604,
+        0x080804bd,
+        0xc40088cf,
+        0x0bf40289,
+        0x8500b51f,
+        0x957e580e,
+        0x09980000,
+        0x0096b085,
+        0x000d0bf4,
+        0x0009f634,
+        0x09b504bd,
+/* 0x0125: intr_skip_watchdog */
+        0x0089e484,
+        0x360bf408,
+        0xcf068849,
+        0x9ac40099,
+        0x220bf402,
+        0xcf04c04c,
+        0xc0f900cc,
+        0xf14f484e,
+        0x0d5453e3,
+        0x023f7e00,
+        0x40c0fc00,
+        0x0cf604c0,
+/* 0x0157: intr_subintr_skip_fifo */
+        0x4004bd00,
+        0x09f60688,
+/* 0x015f: intr_skip_subintr */
+        0xc404bd00,
+        0x0bf42089,
+        0xbfa4f107,
+/* 0x0169: intr_skip_pause */
+        0x4089c4ff,
+        0xf1070bf4,
+/* 0x0173: intr_skip_user0 */
+        0x00ffbfa4,
+        0x0008f604,
+        0x80fc04bd,
+        0xfc0088fe,
+        0xfce0fcf0,
+        0xfcc0fcd0,
+        0xfca0fcb0,
+        0xfc80fc90,
+        0x0032f400,
+/* 0x0196: timer */
+        0x32f401f8,
+        0x03f89810,
+        0xf40086b0,
+        0xfeb53a1c,
+        0xf6380003,
+        0x04bd0008,
+        0x88cf0808,
+        0x0284f000,
+        0x081c1bf4,
+        0x0088cf34,
+        0x0bf4e0a6,
+        0xf4e8a608,
+/* 0x01c6: timer_reset */
+        0x3400161e,
+        0xbd000ef6,
+        0x840eb504,
+/* 0x01d0: timer_enable */
+        0x38000108,
+        0xbd0008f6,
+/* 0x01d9: timer_done */
+        0x1031f404,
+/* 0x01de: send_proc */
+        0x80f900f8,
+        0xe89890f9,
+        0x04e99805,
+        0xa60486f0,
+        0x2a0bf489,
+        0x940398c4,
+        0x80b60488,
+        0x008ebb18,
+        0xb500fa98,
+        0x8db5008a,
+        0x028cb501,
+        0xb6038bb5,
+        0x94f00190,
+        0x04e9b507,
+/* 0x0217: send_done */
+        0xfc0231f4,
+        0xf880fc90,
+/* 0x021d: find */
+        0x0880f900,
+        0x0131f458,
+/* 0x0224: find_loop */
+        0xa6008a98,
+        0x100bf4ae,
+        0xb15880b6,
+        0xf4021086,
+        0x32f4f11b,
+/* 0x0239: find_done */
+        0xfc8eb201,
+/* 0x023f: send */
+        0x7e00f880,
+        0xf400021d,
+        0x00f89b01,
+/* 0x0248: recv */
+        0x9805e898,
+        0x32f404e9,
+        0xf489a601,
+        0x89c43c0b,
+        0x0180b603,
+        0xb50784f0,
+        0xea9805e8,
+        0xfef0f902,
+        0xf0f9018f,
+        0x9994efb2,
+        0x00e9bb04,
+        0x9818e0b6,
+        0xec9803eb,
+        0x01ed9802,
+        0xf900ee98,
+        0xfef0fca5,
+        0x31f400f8,
+/* 0x028f: recv_done */
+        0xf8f0fc01,
+/* 0x0291: init */
+        0x01084100,
+        0xe70011cf,
+        0xb6010911,
+        0x14fe0814,
+        0x00e04100,
+        0x000013f0,
+        0x0001f61c,
+        0xff0104bd,
+        0x01f61400,
+        0x0104bd00,
+        0x0015f102,
+        0xf6100008,
+        0x04bd0001,
+        0xf000d241,
+        0x10fe0013,
+        0x1031f400,
+        0x38000101,
+        0xbd0001f6,
+/* 0x02db: init_proc */
+        0x98580f04,
+        0x16b001f1,
+        0xfa0bf400,
+        0xf0b615f9,
+        0xf20ef458,
+/* 0x02ec: host_send */
+        0xcf04b041,
+        0xa0420011,
+        0x0022cf04,
+        0x0bf412a6,
+        0x071ec42e,
+        0xb704ee94,
+        0x980218e0,
+        0xec9803eb,
+        0x01ed9802,
+        0x7e00ee98,
+        0xb600023f,
+        0x1ec40110,
+        0x04b0400f,
+        0xbd0001f6,
+        0xc70ef404,
+/* 0x0328: host_send_done */
+/* 0x032a: host_recv */
+        0x494100f8,
+        0x5413f14e,
+        0xf4e1a652,
+/* 0x0336: host_recv_wait */
+        0xcc41b90b,
+        0x0011cf04,
+        0xcf04c842,
+        0x16f00022,
+        0xf412a608,
+        0x23c4ef0b,
+        0x0434b607,
+        0x029830b7,
+        0xb5033bb5,
+        0x3db5023c,
+        0x003eb501,
+        0xf00120b6,
+        0xc8400f24,
+        0x0002f604,
+        0x400204bd,
+        0x02f60000,
+        0xf804bd00,
+/* 0x0379: host_init */
+        0x00804100,
+        0xf11014b6,
+        0x40021815,
+        0x01f604d0,
+        0x4104bd00,
+        0x14b60080,
+        0x9815f110,
+        0x04dc4002,
+        0xbd0001f6,
+        0x40010104,
+        0x01f604c4,
+        0xf804bd00,
+/* 0x03a9: memx_func_enter */
+        0x40040600,
+        0x06f607e0,
+/* 0x03b3: memx_func_enter_wait */
+        0x4604bd00,
+        0x66cf07c0,
+        0x0464f000,
+        0x98f70bf4,
+        0x10b60016,
+/* 0x03c7: memx_func_leave */
+        0x0600f804,
+        0x07e44004,
+        0xbd0006f6,
+/* 0x03d1: memx_func_leave_wait */
+        0x07c04604,
+        0xf00066cf,
+        0x1bf40464,
+/* 0x03df: memx_func_wr32 */
+        0x9800f8f7,
+        0x15980016,
+        0x0810b601,
+        0x50f960f9,
+        0xe0fcd0fc,
+        0x00002e7e,
+        0x140003f1,
+        0xa00506fd,
+        0xb604bd05,
+        0x1bf40242,
+/* 0x0407: memx_func_wait */
+        0x0800f8dd,
+        0x0088cf2c,
+        0x98001e98,
+        0x1c98011d,
+        0x031b9802,
+        0x7e1010b6,
+        0xf8000071,
+/* 0x0421: memx_func_delay */
+        0x001e9800,
+        0x7e0410b6,
+        0xf800005d,
+/* 0x042d: memx_exec */
+        0xf9e0f900,
+        0xb2c1b2d0,
+/* 0x0435: memx_exec_next */
+        0x001398b2,
+        0x950410b6,
+        0x30f01034,
+        0xc835980c,
+        0x12a655f9,
+        0xfced1ef4,
+        0x7ee0fcd0,
+        0xf800023f,
+/* 0x0455: memx_info */
+        0x03544c00,
+        0x7e08004b,
+        0xf800023f,
+/* 0x0461: memx_recv */
+        0x01d6b000,
+        0xb0c90bf4,
+        0x0bf400d6,
+/* 0x046f: memx_init */
+        0xf800f8eb,
+/* 0x0471: perf_recv */
+/* 0x0473: perf_init */
+        0xf800f800,
+/* 0x0475: test_recv */
+        0x04584100,
+        0xb60011cf,
+        0x58400110,
+        0x0001f604,
+        0xe7f104bd,
+        0xe3f1d900,
+        0x967e134f,
+        0x00f80001,
+/* 0x0494: test_init */
+        0x7e08004e,
+        0xf8000196,
+/* 0x049d: idle_recv */
+/* 0x049f: idle */
+        0xf400f800,
+        0x54410031,
+        0x0011cf04,
+        0x400110b6,
+        0x01f60454,
+/* 0x04b3: idle_loop */
+        0x0104bd00,
+        0x0232f458,
+/* 0x04b8: idle_proc */
+/* 0x04b8: idle_proc_exec */
+        0x1eb210f9,
+        0x0002487e,
+        0x11f410fc,
+        0x0231f409,
+/* 0x04cb: idle_proc_next */
+        0xb6f00ef4,
+        0x1fa65810,
+        0xf4e81bf4,
+        0x28f4e002,
+        0xc60ef400,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc
new file mode 100644
index 000000000000..6fde0b89e5aa
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#define NVKM_PPWR_CHIPSET GT215
+
+//#define NVKM_FALCON_PC24
+//#define NVKM_FALCON_UNSHIFTED_IO
+//#define NVKM_FALCON_MMIO_UAS
+//#define NVKM_FALCON_MMIO_TRAP
+
+#include "macros.fuc"
+
+.section #nva3_pwr_data
+#define INCLUDE_PROC
+#include "kernel.fuc"
+#include "host.fuc"
+#include "memx.fuc"
+#include "perf.fuc"
+#include "test.fuc"
+#include "idle.fuc"
+#undef INCLUDE_PROC
+
+#define INCLUDE_DATA
+#include "kernel.fuc"
+#include "host.fuc"
+#include "memx.fuc"
+#include "perf.fuc"
+#include "test.fuc"
+#include "idle.fuc"
+#undef INCLUDE_DATA
+.align 256
+
+.section #nva3_pwr_code
+#define INCLUDE_CODE
+#include "kernel.fuc"
+#include "host.fuc"
+#include "memx.fuc"
+#include "perf.fuc"
+#include "test.fuc"
+#include "idle.fuc"
+#undef INCLUDE_CODE
+.align 256
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h
new file mode 100644
index 000000000000..0fa4d7dcd407
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nva3.fuc.h
@@ -0,0 +1,1229 @@
+uint32_t nva3_pwr_data[] = {
+/* 0x0000: proc_kern */
+        0x52544e49,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0058: proc_list_head */
+        0x54534f48,
+        0x00000430,
+        0x000003cd,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x584d454d,
+        0x0000054e,
+        0x00000540,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x46524550,
+        0x00000552,
+        0x00000550,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x54534554,
+        0x0000057b,
+        0x00000554,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x454c4449,
+        0x00000587,
+        0x00000585,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0210: proc_list_tail */
+/* 0x0210: time_prev */
+        0x00000000,
+/* 0x0214: time_next */
+        0x00000000,
+/* 0x0218: fifo_queue */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0298: rfifo_queue */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0318: memx_func_head */
+        0x00010000,
+        0x00000000,
+        0x0000046f,
+/* 0x0324: memx_func_next */
+        0x00000001,
+        0x00000000,
+        0x00000496,
+        0x00000002,
+        0x00000002,
+        0x000004b7,
+        0x00040003,
+        0x00000000,
+        0x000004df,
+        0x00010004,
+        0x00000000,
+        0x000004fc,
+/* 0x0354: memx_func_tail */
+/* 0x0354: memx_data_head */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0b54: memx_data_tail */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
+
+uint32_t nva3_pwr_code[] = {
+        0x030d0ef5,
+/* 0x0004: rd32 */
+        0x07a007f1,
+        0xd00604b6,
+        0x04bd000e,
+        0xf001e7f0,
+        0x07f101e3,
+        0x04b607ac,
+        0x000ed006,
+/* 0x0022: rd32_wait */
+        0xe7f104bd,
+        0xe4b607ac,
+        0x00eecf06,
+        0x7000e4f1,
+        0xf1f21bf4,
+        0xb607a4d7,
+        0xddcf06d4,
+/* 0x003f: wr32 */
+        0xf100f800,
+        0xb607a007,
+        0x0ed00604,
+        0xf104bd00,
+        0xb607a407,
+        0x0dd00604,
+        0xf004bd00,
+        0xe5f002e7,
+        0x01e3f0f0,
+        0x07ac07f1,
+        0xd00604b6,
+        0x04bd000e,
+/* 0x006c: wr32_wait */
+        0x07ace7f1,
+        0xcf06e4b6,
+        0xe4f100ee,
+        0x1bf47000,
+/* 0x007f: nsec */
+        0xf000f8f2,
+        0x84b62c87,
+        0x0088cf06,
+/* 0x0088: nsec_loop */
+        0xb62c97f0,
+        0x99cf0694,
+        0x0298bb00,
+        0xf4069eb8,
+        0x00f8f11e,
+/* 0x009c: wait */
+        0xb62c87f0,
+        0x88cf0684,
+/* 0x00a5: wait_loop */
+        0x02eeb900,
+        0xb90421f4,
+        0xadfd02da,
+        0x06acb804,
+        0xf0150bf4,
+        0x94b62c97,
+        0x0099cf06,
+        0xb80298bb,
+        0x1ef4069b,
+/* 0x00c9: wait_done */
+/* 0x00cb: intr_watchdog */
+        0x9800f8df,
+        0x96b003e9,
+        0x2a0bf400,
+        0xbb840a98,
+        0x1cf4029a,
+        0x01d7f00f,
+        0x025421f5,
+        0x0ef494bd,
+/* 0x00e9: intr_watchdog_next_time */
+        0x850a9815,
+        0xf400a6b0,
+        0x9ab8090b,
+        0x061cf406,
+/* 0x00f8: intr_watchdog_next_time_set */
+/* 0x00fb: intr_watchdog_next_proc */
+        0x80850980,
+        0xe0b603e9,
+        0x10e6b158,
+        0xc61bf402,
+/* 0x010a: intr */
+        0x00f900f8,
+        0x80f904bd,
+        0xa0f990f9,
+        0xc0f9b0f9,
+        0xe0f9d0f9,
+        0xf7f0f0f9,
+        0x0188fe00,
+        0x87f180f9,
+        0x84b605d0,
+        0x0088cf06,
+        0xf10180b6,
+        0xb605d007,
+        0x08d00604,
+        0xf004bd00,
+        0x84b60887,
+        0x0088cf06,
+        0xf40289c4,
+        0x0080230b,
+        0x58e7f085,
+        0x98cb21f4,
+        0x96b08509,
+        0x110bf400,
+        0xb63407f0,
+        0x09d00604,
+        0x8004bd00,
+/* 0x016e: intr_skip_watchdog */
+        0x89e48409,
+        0x0bf40800,
+        0x8897f148,
+        0x0694b606,
+        0xc40099cf,
+        0x0bf4029a,
+        0xc0c7f12c,
+        0x06c4b604,
+        0xf900cccf,
+        0x48e7f1c0,
+        0x53e3f14f,
+        0x00d7f054,
+        0x02b921f5,
+        0x07f1c0fc,
+        0x04b604c0,
+        0x000cd006,
+/* 0x01ae: intr_subintr_skip_fifo */
+        0x07f104bd,
+        0x04b60688,
+        0x0009d006,
+/* 0x01ba: intr_skip_subintr */
+        0x89c404bd,
+        0x070bf420,
+        0xffbfa4f1,
+/* 0x01c4: intr_skip_pause */
+        0xf44089c4,
+        0xa4f1070b,
+/* 0x01ce: intr_skip_user0 */
+        0x07f0ffbf,
+        0x0604b604,
+        0xbd0008d0,
+        0xfe80fc04,
+        0xf0fc0088,
+        0xd0fce0fc,
+        0xb0fcc0fc,
+        0x90fca0fc,
+        0x00fc80fc,
+        0xf80032f4,
+/* 0x01f5: timer */
+        0x1032f401,
+        0xb003f898,
+        0x1cf40086,
+        0x03fe8051,
+        0xb63807f0,
+        0x08d00604,
+        0xf004bd00,
+        0x84b60887,
+        0x0088cf06,
+        0xf40284f0,
+        0x87f0261b,
+        0x0684b634,
+        0xb80088cf,
+        0x0bf406e0,
+        0x06e8b809,
+/* 0x0233: timer_reset */
+        0xf01f1ef4,
+        0x04b63407,
+        0x000ed006,
+        0x0e8004bd,
+/* 0x0241: timer_enable */
+        0x0187f084,
+        0xb63807f0,
+        0x08d00604,
+/* 0x024f: timer_done */
+        0xf404bd00,
+        0x00f81031,
+/* 0x0254: send_proc */
+        0x90f980f9,
+        0x9805e898,
+        0x86f004e9,
+        0x0689b804,
+        0xc42a0bf4,
+        0x88940398,
+        0x1880b604,
+        0x98008ebb,
+        0x8a8000fa,
+        0x018d8000,
+        0x80028c80,
+        0x90b6038b,
+        0x0794f001,
+        0xf404e980,
+/* 0x028e: send_done */
+        0x90fc0231,
+        0x00f880fc,
+/* 0x0294: find */
+        0x87f080f9,
+        0x0131f458,
+/* 0x029c: find_loop */
+        0xb8008a98,
+        0x0bf406ae,
+        0x5880b610,
+        0x021086b1,
+        0xf4f01bf4,
+/* 0x02b2: find_done */
+        0x8eb90132,
+        0xf880fc02,
+/* 0x02b9: send */
+        0x9421f500,
+        0x9701f402,
+/* 0x02c2: recv */
+        0xe89800f8,
+        0x04e99805,
+        0xb80132f4,
+        0x0bf40689,
+        0x0389c43d,
+        0xf00180b6,
+        0xe8800784,
+        0x02ea9805,
+        0x8ffef0f9,
+        0xb9f0f901,
+        0x999402ef,
+        0x00e9bb04,
+        0x9818e0b6,
+        0xec9803eb,
+        0x01ed9802,
+        0xf900ee98,
+        0xfef0fca5,
+        0x31f400f8,
+/* 0x030b: recv_done */
+        0xf8f0fc01,
+/* 0x030d: init */
+        0x0817f100,
+        0x0614b601,
+        0xe70011cf,
+        0xb6010911,
+        0x14fe0814,
+        0xe017f100,
+        0x0013f000,
+        0xb61c07f0,
+        0x01d00604,
+        0xf004bd00,
+        0x07f0ff17,
+        0x0604b614,
+        0xbd0001d0,
+        0x0217f004,
+        0x080015f1,
+        0xb61007f0,
+        0x01d00604,
+        0xf104bd00,
+        0xf0010a17,
+        0x10fe0013,
+        0x1031f400,
+        0xf00117f0,
+        0x04b63807,
+        0x0001d006,
+        0xf7f004bd,
+/* 0x0371: init_proc */
+        0x01f19858,
+        0xf40016b0,
+        0x15f9fa0b,
+        0xf458f0b6,
+/* 0x0382: host_send */
+        0x17f1f20e,
+        0x14b604b0,
+        0x0011cf06,
+        0x04a027f1,
+        0xcf0624b6,
+        0x12b80022,
+        0x320bf406,
+        0x94071ec4,
+        0xe0b704ee,
+        0xeb980218,
+        0x02ec9803,
+        0x9801ed98,
+        0x21f500ee,
+        0x10b602b9,
+        0x0f1ec401,
+        0x04b007f1,
+        0xd00604b6,
+        0x04bd0001,
+/* 0x03cb: host_send_done */
+        0xf8ba0ef4,
+/* 0x03cd: host_recv */
+        0x4917f100,
+        0x5413f14e,
+        0x06e1b852,
+/* 0x03db: host_recv_wait */
+        0xf1aa0bf4,
+        0xb604cc17,
+        0x11cf0614,
+        0xc827f100,
+        0x0624b604,
+        0xf00022cf,
+        0x12b80816,
+        0xe60bf406,
+        0xb60723c4,
+        0x30b70434,
+        0x3b800298,
+        0x023c8003,
+        0x80013d80,
+        0x20b6003e,
+        0x0f24f001,
+        0x04c807f1,
+        0xd00604b6,
+        0x04bd0002,
+        0xf04027f0,
+        0x04b60007,
+        0x0002d006,
+        0x00f804bd,
+/* 0x0430: host_init */
+        0x008017f1,
+        0xf11014b6,
+        0xf1021815,
+        0xb604d007,
+        0x01d00604,
+        0xf104bd00,
+        0xb6008017,
+        0x15f11014,
+        0x07f10298,
+        0x04b604dc,
+        0x0001d006,
+        0x17f004bd,
+        0xc407f101,
+        0x0604b604,
+        0xbd0001d0,
+/* 0x046f: memx_func_enter */
+        0xf000f804,
+        0x07f10467,
+        0x04b607e0,
+        0x0006d006,
+/* 0x047e: memx_func_enter_wait */
+        0x67f104bd,
+        0x64b607c0,
+        0x0066cf06,
+        0xf40464f0,
+        0x1698f30b,
+        0x0410b600,
+/* 0x0496: memx_func_leave */
+        0x67f000f8,
+        0xe407f104,
+        0x0604b607,
+        0xbd0006d0,
+/* 0x04a5: memx_func_leave_wait */
+        0xc067f104,
+        0x0664b607,
+        0xf00066cf,
+        0x1bf40464,
+/* 0x04b7: memx_func_wr32 */
+        0x9800f8f3,
+        0x15980016,
+        0x0810b601,
+        0x50f960f9,
+        0xe0fcd0fc,
+        0xf13f21f4,
+        0xfd140003,
+        0x05800506,
+        0xb604bd00,
+        0x1bf40242,
+/* 0x04df: memx_func_wait */
+        0xf000f8dd,
+        0x84b62c87,
+        0x0088cf06,
+        0x98001e98,
+        0x1c98011d,
+        0x031b9802,
+        0xf41010b6,
+        0x00f89c21,
+/* 0x04fc: memx_func_delay */
+        0xb6001e98,
+        0x21f40410,
+/* 0x0507: memx_exec */
+        0xf900f87f,
+        0xb9d0f9e0,
+        0xb2b902c1,
+/* 0x0511: memx_exec_next */
+        0x00139802,
+        0x950410b6,
+        0x30f01034,
+        0xc835980c,
+        0x12b855f9,
+        0xec1ef406,
+        0xe0fcd0fc,
+        0x02b921f5,
+/* 0x0532: memx_info */
+        0xc7f100f8,
+        0xb7f10354,
+        0x21f50800,
+        0x00f802b9,
+/* 0x0540: memx_recv */
+        0xf401d6b0,
+        0xd6b0c40b,
+        0xe90bf400,
+/* 0x054e: memx_init */
+        0x00f800f8,
+/* 0x0550: perf_recv */
+/* 0x0552: perf_init */
+        0x00f800f8,
+/* 0x0554: test_recv */
+        0x05d817f1,
+        0xcf0614b6,
+        0x10b60011,
+        0xd807f101,
+        0x0604b605,
+        0xbd0001d0,
+        0x00e7f104,
+        0x4fe3f1d9,
+        0xf521f513,
+/* 0x057b: test_init */
+        0xf100f801,
+        0xf50800e7,
+        0xf801f521,
+/* 0x0585: idle_recv */
+/* 0x0587: idle */
+        0xf400f800,
+        0x17f10031,
+        0x14b605d4,
+        0x0011cf06,
+        0xf10110b6,
+        0xb605d407,
+        0x01d00604,
+/* 0x05a3: idle_loop */
+        0xf004bd00,
+        0x32f45817,
+/* 0x05a9: idle_proc */
+/* 0x05a9: idle_proc_exec */
+        0xb910f902,
+        0x21f5021e,
+        0x10fc02c2,
+        0xf40911f4,
+        0x0ef40231,
+/* 0x05bd: idle_proc_next */
+        0x5810b6ef,
+        0xf4061fb8,
+        0x02f4e61b,
+        0x0028f4dd,
+        0x00bb0ef4,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc
new file mode 100644
index 000000000000..eaa64da68e36
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#define NVKM_PPWR_CHIPSET GF100
+
+//#define NVKM_FALCON_PC24
+//#define NVKM_FALCON_UNSHIFTED_IO
+//#define NVKM_FALCON_MMIO_UAS
+//#define NVKM_FALCON_MMIO_TRAP
+
+#include "macros.fuc"
+
+.section #nvc0_pwr_data
+#define INCLUDE_PROC
+#include "kernel.fuc"
+#include "host.fuc"
+#include "memx.fuc"
+#include "perf.fuc"
+#include "test.fuc"
+#include "idle.fuc"
+#undef INCLUDE_PROC
+
+#define INCLUDE_DATA
+#include "kernel.fuc"
+#include "host.fuc"
+#include "memx.fuc"
+#include "perf.fuc"
+#include "test.fuc"
+#include "idle.fuc"
+#undef INCLUDE_DATA
+.align 256
+
+.section #nvc0_pwr_code
+#define INCLUDE_CODE
+#include "kernel.fuc"
+#include "host.fuc"
+#include "memx.fuc"
+#include "perf.fuc"
+#include "test.fuc"
+#include "idle.fuc"
+#undef INCLUDE_CODE
+.align 256
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h
new file mode 100644
index 000000000000..82c8e8b88917
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvc0.fuc.h
@@ -0,0 +1,1229 @@
+uint32_t nvc0_pwr_data[] = {
+/* 0x0000: proc_kern */
+        0x52544e49,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0058: proc_list_head */
+        0x54534f48,
+        0x00000430,
+        0x000003cd,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x584d454d,
+        0x0000054e,
+        0x00000540,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x46524550,
+        0x00000552,
+        0x00000550,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x54534554,
+        0x0000057b,
+        0x00000554,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x454c4449,
+        0x00000587,
+        0x00000585,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0210: proc_list_tail */
+/* 0x0210: time_prev */
+        0x00000000,
+/* 0x0214: time_next */
+        0x00000000,
+/* 0x0218: fifo_queue */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0298: rfifo_queue */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0318: memx_func_head */
+        0x00010000,
+        0x00000000,
+        0x0000046f,
+/* 0x0324: memx_func_next */
+        0x00000001,
+        0x00000000,
+        0x00000496,
+        0x00000002,
+        0x00000002,
+        0x000004b7,
+        0x00040003,
+        0x00000000,
+        0x000004df,
+        0x00010004,
+        0x00000000,
+        0x000004fc,
+/* 0x0354: memx_func_tail */
+/* 0x0354: memx_data_head */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0b54: memx_data_tail */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
+
+uint32_t nvc0_pwr_code[] = {
+        0x030d0ef5,
+/* 0x0004: rd32 */
+        0x07a007f1,
+        0xd00604b6,
+        0x04bd000e,
+        0xf001e7f0,
+        0x07f101e3,
+        0x04b607ac,
+        0x000ed006,
+/* 0x0022: rd32_wait */
+        0xe7f104bd,
+        0xe4b607ac,
+        0x00eecf06,
+        0x7000e4f1,
+        0xf1f21bf4,
+        0xb607a4d7,
+        0xddcf06d4,
+/* 0x003f: wr32 */
+        0xf100f800,
+        0xb607a007,
+        0x0ed00604,
+        0xf104bd00,
+        0xb607a407,
+        0x0dd00604,
+        0xf004bd00,
+        0xe5f002e7,
+        0x01e3f0f0,
+        0x07ac07f1,
+        0xd00604b6,
+        0x04bd000e,
+/* 0x006c: wr32_wait */
+        0x07ace7f1,
+        0xcf06e4b6,
+        0xe4f100ee,
+        0x1bf47000,
+/* 0x007f: nsec */
+        0xf000f8f2,
+        0x84b62c87,
+        0x0088cf06,
+/* 0x0088: nsec_loop */
+        0xb62c97f0,
+        0x99cf0694,
+        0x0298bb00,
+        0xf4069eb8,
+        0x00f8f11e,
+/* 0x009c: wait */
+        0xb62c87f0,
+        0x88cf0684,
+/* 0x00a5: wait_loop */
+        0x02eeb900,
+        0xb90421f4,
+        0xadfd02da,
+        0x06acb804,
+        0xf0150bf4,
+        0x94b62c97,
+        0x0099cf06,
+        0xb80298bb,
+        0x1ef4069b,
+/* 0x00c9: wait_done */
+/* 0x00cb: intr_watchdog */
+        0x9800f8df,
+        0x96b003e9,
+        0x2a0bf400,
+        0xbb840a98,
+        0x1cf4029a,
+        0x01d7f00f,
+        0x025421f5,
+        0x0ef494bd,
+/* 0x00e9: intr_watchdog_next_time */
+        0x850a9815,
+        0xf400a6b0,
+        0x9ab8090b,
+        0x061cf406,
+/* 0x00f8: intr_watchdog_next_time_set */
+/* 0x00fb: intr_watchdog_next_proc */
+        0x80850980,
+        0xe0b603e9,
+        0x10e6b158,
+        0xc61bf402,
+/* 0x010a: intr */
+        0x00f900f8,
+        0x80f904bd,
+        0xa0f990f9,
+        0xc0f9b0f9,
+        0xe0f9d0f9,
+        0xf7f0f0f9,
+        0x0188fe00,
+        0x87f180f9,
+        0x84b605d0,
+        0x0088cf06,
+        0xf10180b6,
+        0xb605d007,
+        0x08d00604,
+        0xf004bd00,
+        0x84b60887,
+        0x0088cf06,
+        0xf40289c4,
+        0x0080230b,
+        0x58e7f085,
+        0x98cb21f4,
+        0x96b08509,
+        0x110bf400,
+        0xb63407f0,
+        0x09d00604,
+        0x8004bd00,
+/* 0x016e: intr_skip_watchdog */
+        0x89e48409,
+        0x0bf40800,
+        0x8897f148,
+        0x0694b606,
+        0xc40099cf,
+        0x0bf4029a,
+        0xc0c7f12c,
+        0x06c4b604,
+        0xf900cccf,
+        0x48e7f1c0,
+        0x53e3f14f,
+        0x00d7f054,
+        0x02b921f5,
+        0x07f1c0fc,
+        0x04b604c0,
+        0x000cd006,
+/* 0x01ae: intr_subintr_skip_fifo */
+        0x07f104bd,
+        0x04b60688,
+        0x0009d006,
+/* 0x01ba: intr_skip_subintr */
+        0x89c404bd,
+        0x070bf420,
+        0xffbfa4f1,
+/* 0x01c4: intr_skip_pause */
+        0xf44089c4,
+        0xa4f1070b,
+/* 0x01ce: intr_skip_user0 */
+        0x07f0ffbf,
+        0x0604b604,
+        0xbd0008d0,
+        0xfe80fc04,
+        0xf0fc0088,
+        0xd0fce0fc,
+        0xb0fcc0fc,
+        0x90fca0fc,
+        0x00fc80fc,
+        0xf80032f4,
+/* 0x01f5: timer */
+        0x1032f401,
+        0xb003f898,
+        0x1cf40086,
+        0x03fe8051,
+        0xb63807f0,
+        0x08d00604,
+        0xf004bd00,
+        0x84b60887,
+        0x0088cf06,
+        0xf40284f0,
+        0x87f0261b,
+        0x0684b634,
+        0xb80088cf,
+        0x0bf406e0,
+        0x06e8b809,
+/* 0x0233: timer_reset */
+        0xf01f1ef4,
+        0x04b63407,
+        0x000ed006,
+        0x0e8004bd,
+/* 0x0241: timer_enable */
+        0x0187f084,
+        0xb63807f0,
+        0x08d00604,
+/* 0x024f: timer_done */
+        0xf404bd00,
+        0x00f81031,
+/* 0x0254: send_proc */
+        0x90f980f9,
+        0x9805e898,
+        0x86f004e9,
+        0x0689b804,
+        0xc42a0bf4,
+        0x88940398,
+        0x1880b604,
+        0x98008ebb,
+        0x8a8000fa,
+        0x018d8000,
+        0x80028c80,
+        0x90b6038b,
+        0x0794f001,
+        0xf404e980,
+/* 0x028e: send_done */
+        0x90fc0231,
+        0x00f880fc,
+/* 0x0294: find */
+        0x87f080f9,
+        0x0131f458,
+/* 0x029c: find_loop */
+        0xb8008a98,
+        0x0bf406ae,
+        0x5880b610,
+        0x021086b1,
+        0xf4f01bf4,
+/* 0x02b2: find_done */
+        0x8eb90132,
+        0xf880fc02,
+/* 0x02b9: send */
+        0x9421f500,
+        0x9701f402,
+/* 0x02c2: recv */
+        0xe89800f8,
+        0x04e99805,
+        0xb80132f4,
+        0x0bf40689,
+        0x0389c43d,
+        0xf00180b6,
+        0xe8800784,
+        0x02ea9805,
+        0x8ffef0f9,
+        0xb9f0f901,
+        0x999402ef,
+        0x00e9bb04,
+        0x9818e0b6,
+        0xec9803eb,
+        0x01ed9802,
+        0xf900ee98,
+        0xfef0fca5,
+        0x31f400f8,
+/* 0x030b: recv_done */
+        0xf8f0fc01,
+/* 0x030d: init */
+        0x0817f100,
+        0x0614b601,
+        0xe70011cf,
+        0xb6010911,
+        0x14fe0814,
+        0xe017f100,
+        0x0013f000,
+        0xb61c07f0,
+        0x01d00604,
+        0xf004bd00,
+        0x07f0ff17,
+        0x0604b614,
+        0xbd0001d0,
+        0x0217f004,
+        0x080015f1,
+        0xb61007f0,
+        0x01d00604,
+        0xf104bd00,
+        0xf0010a17,
+        0x10fe0013,
+        0x1031f400,
+        0xf00117f0,
+        0x04b63807,
+        0x0001d006,
+        0xf7f004bd,
+/* 0x0371: init_proc */
+        0x01f19858,
+        0xf40016b0,
+        0x15f9fa0b,
+        0xf458f0b6,
+/* 0x0382: host_send */
+        0x17f1f20e,
+        0x14b604b0,
+        0x0011cf06,
+        0x04a027f1,
+        0xcf0624b6,
+        0x12b80022,
+        0x320bf406,
+        0x94071ec4,
+        0xe0b704ee,
+        0xeb980218,
+        0x02ec9803,
+        0x9801ed98,
+        0x21f500ee,
+        0x10b602b9,
+        0x0f1ec401,
+        0x04b007f1,
+        0xd00604b6,
+        0x04bd0001,
+/* 0x03cb: host_send_done */
+        0xf8ba0ef4,
+/* 0x03cd: host_recv */
+        0x4917f100,
+        0x5413f14e,
+        0x06e1b852,
+/* 0x03db: host_recv_wait */
+        0xf1aa0bf4,
+        0xb604cc17,
+        0x11cf0614,
+        0xc827f100,
+        0x0624b604,
+        0xf00022cf,
+        0x12b80816,
+        0xe60bf406,
+        0xb60723c4,
+        0x30b70434,
+        0x3b800298,
+        0x023c8003,
+        0x80013d80,
+        0x20b6003e,
+        0x0f24f001,
+        0x04c807f1,
+        0xd00604b6,
+        0x04bd0002,
+        0xf04027f0,
+        0x04b60007,
+        0x0002d006,
+        0x00f804bd,
+/* 0x0430: host_init */
+        0x008017f1,
+        0xf11014b6,
+        0xf1021815,
+        0xb604d007,
+        0x01d00604,
+        0xf104bd00,
+        0xb6008017,
+        0x15f11014,
+        0x07f10298,
+        0x04b604dc,
+        0x0001d006,
+        0x17f004bd,
+        0xc407f101,
+        0x0604b604,
+        0xbd0001d0,
+/* 0x046f: memx_func_enter */
+        0xf000f804,
+        0x07f10467,
+        0x04b607e0,
+        0x0006d006,
+/* 0x047e: memx_func_enter_wait */
+        0x67f104bd,
+        0x64b607c0,
+        0x0066cf06,
+        0xf40464f0,
+        0x1698f30b,
+        0x0410b600,
+/* 0x0496: memx_func_leave */
+        0x67f000f8,
+        0xe407f104,
+        0x0604b607,
+        0xbd0006d0,
+/* 0x04a5: memx_func_leave_wait */
+        0xc067f104,
+        0x0664b607,
+        0xf00066cf,
+        0x1bf40464,
+/* 0x04b7: memx_func_wr32 */
+        0x9800f8f3,
+        0x15980016,
+        0x0810b601,
+        0x50f960f9,
+        0xe0fcd0fc,
+        0xf13f21f4,
+        0xfd140003,
+        0x05800506,
+        0xb604bd00,
+        0x1bf40242,
+/* 0x04df: memx_func_wait */
+        0xf000f8dd,
+        0x84b62c87,
+        0x0088cf06,
+        0x98001e98,
+        0x1c98011d,
+        0x031b9802,
+        0xf41010b6,
+        0x00f89c21,
+/* 0x04fc: memx_func_delay */
+        0xb6001e98,
+        0x21f40410,
+/* 0x0507: memx_exec */
+        0xf900f87f,
+        0xb9d0f9e0,
+        0xb2b902c1,
+/* 0x0511: memx_exec_next */
+        0x00139802,
+        0x950410b6,
+        0x30f01034,
+        0xc835980c,
+        0x12b855f9,
+        0xec1ef406,
+        0xe0fcd0fc,
+        0x02b921f5,
+/* 0x0532: memx_info */
+        0xc7f100f8,
+        0xb7f10354,
+        0x21f50800,
+        0x00f802b9,
+/* 0x0540: memx_recv */
+        0xf401d6b0,
+        0xd6b0c40b,
+        0xe90bf400,
+/* 0x054e: memx_init */
+        0x00f800f8,
+/* 0x0550: perf_recv */
+/* 0x0552: perf_init */
+        0x00f800f8,
+/* 0x0554: test_recv */
+        0x05d817f1,
+        0xcf0614b6,
+        0x10b60011,
+        0xd807f101,
+        0x0604b605,
+        0xbd0001d0,
+        0x00e7f104,
+        0x4fe3f1d9,
+        0xf521f513,
+/* 0x057b: test_init */
+        0xf100f801,
+        0xf50800e7,
+        0xf801f521,
+/* 0x0585: idle_recv */
+/* 0x0587: idle */
+        0xf400f800,
+        0x17f10031,
+        0x14b605d4,
+        0x0011cf06,
+        0xf10110b6,
+        0xb605d407,
+        0x01d00604,
+/* 0x05a3: idle_loop */
+        0xf004bd00,
+        0x32f45817,
+/* 0x05a9: idle_proc */
+/* 0x05a9: idle_proc_exec */
+        0xb910f902,
+        0x21f5021e,
+        0x10fc02c2,
+        0xf40911f4,
+        0x0ef40231,
+/* 0x05bd: idle_proc_next */
+        0x5810b6ef,
+        0xf4061fb8,
+        0x02f4e61b,
+        0x0028f4dd,
+        0x00bb0ef4,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc
new file mode 100644
index 000000000000..32d65ea254dd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#define NVKM_PPWR_CHIPSET GF119
+
+//#define NVKM_FALCON_PC24
+#define NVKM_FALCON_UNSHIFTED_IO
+//#define NVKM_FALCON_MMIO_UAS
+//#define NVKM_FALCON_MMIO_TRAP
+
+#include "macros.fuc"
+
+.section #nvd0_pwr_data
+#define INCLUDE_PROC
+#include "kernel.fuc"
+#include "host.fuc"
+#include "memx.fuc"
+#include "perf.fuc"
+#include "test.fuc"
+#include "idle.fuc"
+#undef INCLUDE_PROC
+
+#define INCLUDE_DATA
+#include "kernel.fuc"
+#include "host.fuc"
+#include "memx.fuc"
+#include "perf.fuc"
+#include "test.fuc"
+#include "idle.fuc"
+#undef INCLUDE_DATA
+.align 256
+
+.section #nvd0_pwr_code
+#define INCLUDE_CODE
+#include "kernel.fuc"
+#include "host.fuc"
+#include "memx.fuc"
+#include "perf.fuc"
+#include "test.fuc"
+#include "idle.fuc"
+#undef INCLUDE_CODE
+.align 256
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h
new file mode 100644
index 000000000000..ce65e2a4b789
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/nvd0.fuc.h
@@ -0,0 +1,1229 @@
+uint32_t nvd0_pwr_data[] = {
+/* 0x0000: proc_kern */
+        0x52544e49,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0058: proc_list_head */
+        0x54534f48,
+        0x000003be,
+        0x00000367,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x584d454d,
+        0x000004c4,
+        0x000004b6,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x46524550,
+        0x000004c8,
+        0x000004c6,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x54534554,
+        0x000004eb,
+        0x000004ca,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x454c4449,
+        0x000004f7,
+        0x000004f5,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0210: proc_list_tail */
+/* 0x0210: time_prev */
+        0x00000000,
+/* 0x0214: time_next */
+        0x00000000,
+/* 0x0218: fifo_queue */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0298: rfifo_queue */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0318: memx_func_head */
+        0x00010000,
+        0x00000000,
+        0x000003f4,
+/* 0x0324: memx_func_next */
+        0x00000001,
+        0x00000000,
+        0x00000415,
+        0x00000002,
+        0x00000002,
+        0x00000430,
+        0x00040003,
+        0x00000000,
+        0x00000458,
+        0x00010004,
+        0x00000000,
+        0x00000472,
+/* 0x0354: memx_func_tail */
+/* 0x0354: memx_data_head */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0b54: memx_data_tail */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
+
+uint32_t nvd0_pwr_code[] = {
+        0x02bf0ef5,
+/* 0x0004: rd32 */
+        0x07a007f1,
+        0xbd000ed0,
+        0x01e7f004,
+        0xf101e3f0,
+        0xd007ac07,
+        0x04bd000e,
+/* 0x001c: rd32_wait */
+        0x07ace7f1,
+        0xf100eecf,
+        0xf47000e4,
+        0xd7f1f51b,
+        0xddcf07a4,
+/* 0x0033: wr32 */
+        0xf100f800,
+        0xd007a007,
+        0x04bd000e,
+        0x07a407f1,
+        0xbd000dd0,
+        0x02e7f004,
+        0xf0f0e5f0,
+        0x07f101e3,
+        0x0ed007ac,
+/* 0x0057: wr32_wait */
+        0xf104bd00,
+        0xcf07ace7,
+        0xe4f100ee,
+        0x1bf47000,
+/* 0x0067: nsec */
+        0xf000f8f5,
+        0x88cf2c87,
+/* 0x006d: nsec_loop */
+        0x2c97f000,
+        0xbb0099cf,
+        0x9eb80298,
+        0xf41ef406,
+/* 0x007e: wait */
+        0x87f000f8,
+        0x0088cf2c,
+/* 0x0084: wait_loop */
+        0xf402eeb9,
+        0xdab90421,
+        0x04adfd02,
+        0xf406acb8,
+        0x97f0120b,
+        0x0099cf2c,
+        0xb80298bb,
+        0x1ef4069b,
+/* 0x00a5: wait_done */
+/* 0x00a7: intr_watchdog */
+        0x9800f8e2,
+        0x96b003e9,
+        0x2a0bf400,
+        0xbb840a98,
+        0x1cf4029a,
+        0x01d7f00f,
+        0x020621f5,
+        0x0ef494bd,
+/* 0x00c5: intr_watchdog_next_time */
+        0x850a9815,
+        0xf400a6b0,
+        0x9ab8090b,
+        0x061cf406,
+/* 0x00d4: intr_watchdog_next_time_set */
+/* 0x00d7: intr_watchdog_next_proc */
+        0x80850980,
+        0xe0b603e9,
+        0x10e6b158,
+        0xc61bf402,
+/* 0x00e6: intr */
+        0x00f900f8,
+        0x80f904bd,
+        0xa0f990f9,
+        0xc0f9b0f9,
+        0xe0f9d0f9,
+        0xf7f0f0f9,
+        0x0188fe00,
+        0x87f180f9,
+        0x88cf05d0,
+        0x0180b600,
+        0x05d007f1,
+        0xbd0008d0,
+        0x0887f004,
+        0xc40088cf,
+        0x0bf40289,
+        0x85008020,
+        0xf458e7f0,
+        0x0998a721,
+        0x0096b085,
+        0xf00e0bf4,
+        0x09d03407,
+        0x8004bd00,
+/* 0x013e: intr_skip_watchdog */
+        0x89e48409,
+        0x0bf40800,
+        0x8897f13c,
+        0x0099cf06,
+        0xf4029ac4,
+        0xc7f1260b,
+        0xcccf04c0,
+        0xf1c0f900,
+        0xf14f48e7,
+        0xf05453e3,
+        0x21f500d7,
+        0xc0fc026b,
+        0x04c007f1,
+        0xbd000cd0,
+/* 0x0175: intr_subintr_skip_fifo */
+        0x8807f104,
+        0x0009d006,
+/* 0x017e: intr_skip_subintr */
+        0x89c404bd,
+        0x070bf420,
+        0xffbfa4f1,
+/* 0x0188: intr_skip_pause */
+        0xf44089c4,
+        0xa4f1070b,
+/* 0x0192: intr_skip_user0 */
+        0x07f0ffbf,
+        0x0008d004,
+        0x80fc04bd,
+        0xfc0088fe,
+        0xfce0fcf0,
+        0xfcc0fcd0,
+        0xfca0fcb0,
+        0xfc80fc90,
+        0x0032f400,
+/* 0x01b6: timer */
+        0x32f401f8,
+        0x03f89810,
+        0xf40086b0,
+        0xfe80421c,
+        0x3807f003,
+        0xbd0008d0,
+        0x0887f004,
+        0xf00088cf,
+        0x1bf40284,
+        0x3487f020,
+        0xb80088cf,
+        0x0bf406e0,
+        0x06e8b809,
+/* 0x01eb: timer_reset */
+        0xf0191ef4,
+        0x0ed03407,
+        0x8004bd00,
+/* 0x01f6: timer_enable */
+        0x87f0840e,
+        0x3807f001,
+        0xbd0008d0,
+/* 0x0201: timer_done */
+        0x1031f404,
+/* 0x0206: send_proc */
+        0x80f900f8,
+        0xe89890f9,
+        0x04e99805,
+        0xb80486f0,
+        0x0bf40689,
+        0x0398c42a,
+        0xb6048894,
+        0x8ebb1880,
+        0x00fa9800,
+        0x80008a80,
+        0x8c80018d,
+        0x038b8002,
+        0xf00190b6,
+        0xe9800794,
+        0x0231f404,
+/* 0x0240: send_done */
+        0x80fc90fc,
+/* 0x0246: find */
+        0x80f900f8,
+        0xf45887f0,
+/* 0x024e: find_loop */
+        0x8a980131,
+        0x06aeb800,
+        0xb6100bf4,
+        0x86b15880,
+        0x1bf40210,
+        0x0132f4f0,
+/* 0x0264: find_done */
+        0xfc028eb9,
+/* 0x026b: send */
+        0xf500f880,
+        0xf4024621,
+        0x00f89701,
+/* 0x0274: recv */
+        0x9805e898,
+        0x32f404e9,
+        0x0689b801,
+        0xc43d0bf4,
+        0x80b60389,
+        0x0784f001,
+        0x9805e880,
+        0xf0f902ea,
+        0xf9018ffe,
+        0x02efb9f0,
+        0xbb049994,
+        0xe0b600e9,
+        0x03eb9818,
+        0x9802ec98,
+        0xee9801ed,
+        0xfca5f900,
+        0x00f8fef0,
+        0xfc0131f4,
+/* 0x02bd: recv_done */
+/* 0x02bf: init */
+        0xf100f8f0,
+        0xcf010817,
+        0x11e70011,
+        0x14b60109,
+        0x0014fe08,
+        0x00e017f1,
+        0xf00013f0,
+        0x01d01c07,
+        0xf004bd00,
+        0x07f0ff17,
+        0x0001d014,
+        0x17f004bd,
+        0x0015f102,
+        0x1007f008,
+        0xbd0001d0,
+        0xe617f104,
+        0x0013f000,
+        0xf40010fe,
+        0x17f01031,
+        0x3807f001,
+        0xbd0001d0,
+        0x58f7f004,
+/* 0x0314: init_proc */
+        0xb001f198,
+        0x0bf40016,
+        0xb615f9fa,
+        0x0ef458f0,
+/* 0x0325: host_send */
+        0xb017f1f2,
+        0x0011cf04,
+        0x04a027f1,
+        0xb80022cf,
+        0x0bf40612,
+        0x071ec42f,
+        0xb704ee94,
+        0x980218e0,
+        0xec9803eb,
+        0x01ed9802,
+        0xf500ee98,
+        0xb6026b21,
+        0x1ec40110,
+        0xb007f10f,
+        0x0001d004,
+        0x0ef404bd,
+/* 0x0365: host_send_done */
+/* 0x0367: host_recv */
+        0xf100f8c3,
+        0xf14e4917,
+        0xb8525413,
+        0x0bf406e1,
+/* 0x0375: host_recv_wait */
+        0xcc17f1b3,
+        0x0011cf04,
+        0x04c827f1,
+        0xf00022cf,
+        0x12b80816,
+        0xec0bf406,
+        0xb60723c4,
+        0x30b70434,
+        0x3b800298,
+        0x023c8003,
+        0x80013d80,
+        0x20b6003e,
+        0x0f24f001,
+        0x04c807f1,
+        0xbd0002d0,
+        0x4027f004,
+        0xd00007f0,
+        0x04bd0002,
+/* 0x03be: host_init */
+        0x17f100f8,
+        0x14b60080,
+        0x1815f110,
+        0xd007f102,
+        0x0001d004,
+        0x17f104bd,
+        0x14b60080,
+        0x9815f110,
+        0xdc07f102,
+        0x0001d004,
+        0x17f004bd,
+        0xc407f101,
+        0x0001d004,
+        0x00f804bd,
+/* 0x03f4: memx_func_enter */
+        0xf10467f0,
+        0xd007e007,
+        0x04bd0006,
+/* 0x0400: memx_func_enter_wait */
+        0x07c067f1,
+        0xf00066cf,
+        0x0bf40464,
+        0x001698f6,
+        0xf80410b6,
+/* 0x0415: memx_func_leave */
+        0x0467f000,
+        0x07e407f1,
+        0xbd0006d0,
+/* 0x0421: memx_func_leave_wait */
+        0xc067f104,
+        0x0066cf07,
+        0xf40464f0,
+        0x00f8f61b,
+/* 0x0430: memx_func_wr32 */
+        0x98001698,
+        0x10b60115,
+        0xf960f908,
+        0xfcd0fc50,
+        0x3321f4e0,
+        0x140003f1,
+        0x800506fd,
+        0x04bd0005,
+        0xf40242b6,
+        0x00f8dd1b,
+/* 0x0458: memx_func_wait */
+        0xcf2c87f0,
+        0x1e980088,
+        0x011d9800,
+        0x98021c98,
+        0x10b6031b,
+        0x7e21f410,
+/* 0x0472: memx_func_delay */
+        0x1e9800f8,
+        0x0410b600,
+        0xf86721f4,
+/* 0x047d: memx_exec */
+        0xf9e0f900,
+        0x02c1b9d0,
+/* 0x0487: memx_exec_next */
+        0x9802b2b9,
+        0x10b60013,
+        0x10349504,
+        0x980c30f0,
+        0x55f9c835,
+        0xf40612b8,
+        0xd0fcec1e,
+        0x21f5e0fc,
+        0x00f8026b,
+/* 0x04a8: memx_info */
+        0x0354c7f1,
+        0x0800b7f1,
+        0x026b21f5,
+/* 0x04b6: memx_recv */
+        0xd6b000f8,
+        0xc40bf401,
+        0xf400d6b0,
+        0x00f8e90b,
+/* 0x04c4: memx_init */
+/* 0x04c6: perf_recv */
+        0x00f800f8,
+/* 0x04c8: perf_init */
+/* 0x04ca: test_recv */
+        0x17f100f8,
+        0x11cf05d8,
+        0x0110b600,
+        0x05d807f1,
+        0xbd0001d0,
+        0x00e7f104,
+        0x4fe3f1d9,
+        0xb621f513,
+/* 0x04eb: test_init */
+        0xf100f801,
+        0xf50800e7,
+        0xf801b621,
+/* 0x04f5: idle_recv */
+/* 0x04f7: idle */
+        0xf400f800,
+        0x17f10031,
+        0x11cf05d4,
+        0x0110b600,
+        0x05d407f1,
+        0xbd0001d0,
+/* 0x050d: idle_loop */
+        0x5817f004,
+/* 0x0513: idle_proc */
+/* 0x0513: idle_proc_exec */
+        0xf90232f4,
+        0x021eb910,
+        0x027421f5,
+        0x11f410fc,
+        0x0231f409,
+/* 0x0527: idle_proc_next */
+        0xb6ef0ef4,
+        0x1fb85810,
+        0xe61bf406,
+        0xf4dd02f4,
+        0x0ef40028,
+        0x000000c1,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/os.h b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/os.h
new file mode 100644
index 000000000000..5fb0cccc6c64
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/os.h
@@ -0,0 +1,27 @@
+#ifndef __NVKM_PWR_OS_H__
+#define __NVKM_PWR_OS_H__
+
+/* Process names */
+#define PROC_KERN 0x52544e49
+#define PROC_IDLE 0x454c4449
+#define PROC_HOST 0x54534f48
+#define PROC_MEMX 0x584d454d
+#define PROC_PERF 0x46524550
+#define PROC_TEST 0x54534554
+
+/* KERN: message identifiers */
+#define KMSG_FIFO   0x00000000
+#define KMSG_ALARM  0x00000001
+
+/* MEMX: message identifiers */
+#define MEMX_MSG_INFO 0
+#define MEMX_MSG_EXEC 1
+
+/* MEMX: script opcode definitions */
+#define MEMX_ENTER  0
+#define MEMX_LEAVE  1
+#define MEMX_WR32   2
+#define MEMX_WAIT   3
+#define MEMX_DELAY  4
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/perf.fuc b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/perf.fuc
new file mode 100644
index 000000000000..38eadf705cbf
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/perf.fuc
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#ifdef INCLUDE_PROC
+process(PROC_PERF, #perf_init, #perf_recv)
+#endif
+
+/******************************************************************************
+ * PERF data segment
+ *****************************************************************************/
+#ifdef INCLUDE_DATA
+#endif
+
+/******************************************************************************
+ * PERF code segment
+ *****************************************************************************/
+#ifdef INCLUDE_CODE
+
+// description
+//
+// $r15 - current (perf)
+// $r14 - sender process name
+// $r13 - message
+// $r12 - data0
+// $r11 - data1
+// $r0  - zero
+perf_recv:
+        ret
+
+// description
+//
+// $r15 - current (perf)
+// $r0  - zero
+perf_init:
+        ret
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/test.fuc b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/test.fuc
new file mode 100644
index 000000000000..0c3a71bf5459
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/fuc/test.fuc
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#ifdef INCLUDE_PROC
+process(PROC_TEST, #test_init, #test_recv)
+#endif
+
+/******************************************************************************
+ * TEST data segment
+ *****************************************************************************/
+#ifdef INCLUDE_DATA
+#endif
+
+/******************************************************************************
+ * TEST code segment
+ *****************************************************************************/
+#ifdef INCLUDE_CODE
+// description
+//
+// $r15 - current (test)
+// $r14 - sender process name
+// $r13 - message
+// $r12 - data0
+// $r11 - data1
+// $r0  - zero
+test_recv:
+        nv_iord($r1, NV_PPWR_DSCRATCH(2))
+        add b32 $r1 1
+        nv_iowr(NV_PPWR_DSCRATCH(2), $r1)
+        mov $r14 -0x2700 /* 0xd900, envyas grrr! */
+        sethi $r14 0x134f0000
+        call(timer)
+        ret
+
+// description
+//
+// $r15 - current (test)
+// $r0  - zero
+test_init:
+        mov $r14 0x800
+        call(timer)
+        ret
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c
new file mode 100644
index 000000000000..03de3107d29f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/memx.c
@@ -0,0 +1,121 @@
+#ifndef __NVKM_PWR_MEMX_H__
+#define __NVKM_PWR_MEMX_H__
+
+#include <subdev/pwr.h>
+#include <subdev/pwr/fuc/os.h>
+
+struct nouveau_memx {
+        struct nouveau_pwr *ppwr;
+        u32 base;
+        u32 size;
+        struct {
+                u32 mthd;
+                u32 size;
+                u32 data[64];
+        } c;
+};
+
+static void
+memx_out(struct nouveau_memx *memx)
+{
+        struct nouveau_pwr *ppwr = memx->ppwr;
+        int i;
+
+        if (memx->c.size) {
+                nv_wr32(ppwr, 0x10a1c4, (memx->c.size << 16) | memx->c.mthd);
+                for (i = 0; i < memx->c.size; i++)
+                        nv_wr32(ppwr, 0x10a1c4, memx->c.data[i]);
+                memx->c.size = 0;
+        }
+}
+
+static void
+memx_cmd(struct nouveau_memx *memx, u32 mthd, u32 size, u32 data[])
+{
+        if ((memx->c.size + size >= ARRAY_SIZE(memx->c.data)) ||
+            (memx->c.size && memx->c.mthd != mthd))
+                memx_out(memx);
+        memcpy(&memx->c.data[memx->c.size], data, size * sizeof(data[0]));
+        memx->c.size += size;
+        memx->c.mthd  = mthd;
+}
+
+int
+nouveau_memx_init(struct nouveau_pwr *ppwr, struct nouveau_memx **pmemx)
+{
+        struct nouveau_memx *memx;
+        u32 reply[2];
+        int ret;
+
+        ret = ppwr->message(ppwr, reply, PROC_MEMX, MEMX_MSG_INFO, 0, 0);
+        if (ret)
+                return ret;
+
+        memx = *pmemx = kzalloc(sizeof(*memx), GFP_KERNEL);
+        if (!memx)
+                return -ENOMEM;
+        memx->ppwr = ppwr;
+        memx->base = reply[0];
+        memx->size = reply[1];
+
+        /* acquire data segment access */
+        do {
+                nv_wr32(ppwr, 0x10a580, 0x00000003);
+        } while (nv_rd32(ppwr, 0x10a580) != 0x00000003);
+        nv_wr32(ppwr, 0x10a1c0, 0x01000000 | memx->base);
+        nv_wr32(ppwr, 0x10a1c4, 0x00010000 | MEMX_ENTER);
+        nv_wr32(ppwr, 0x10a1c4, 0x00000000);
+        return 0;
+}
+
+int
+nouveau_memx_fini(struct nouveau_memx **pmemx, bool exec)
+{
+        struct nouveau_memx *memx = *pmemx;
+        struct nouveau_pwr *ppwr = memx->ppwr;
+        u32 finish, reply[2];
+
+        /* flush the cache... */
+        memx_out(memx);
+
+        /* release data segment access */
+        nv_wr32(ppwr, 0x10a1c4, 0x00000000 | MEMX_LEAVE);
+        finish = nv_rd32(ppwr, 0x10a1c0) & 0x00ffffff;
+        nv_wr32(ppwr, 0x10a580, 0x00000000);
+
+        /* call MEMX process to execute the script, and wait for reply */
+        if (exec) {
+                ppwr->message(ppwr, reply, PROC_MEMX, MEMX_MSG_EXEC,
+                                 memx->base, finish);
+        }
+
+        kfree(memx);
+        return 0;
+}
+
+void
+nouveau_memx_wr32(struct nouveau_memx *memx, u32 addr, u32 data)
+{
+        nv_debug(memx->ppwr, "R[%06x] = 0x%08x\n", addr, data);
+        memx_cmd(memx, MEMX_WR32, 2, (u32[]){ addr, data });
+}
+
+void
+nouveau_memx_wait(struct nouveau_memx *memx,
+                  u32 addr, u32 mask, u32 data, u32 nsec)
+{
+        nv_debug(memx->ppwr, "R[%06x] & 0x%08x == 0x%08x, %d us\n",
+                                addr, mask, data, nsec);
+        memx_cmd(memx, MEMX_WAIT, 4, (u32[]){ addr, ~mask, data, nsec });
+        memx_out(memx); /* fuc can't handle multiple */
+}
+
+void
+nouveau_memx_nsec(struct nouveau_memx *memx, u32 nsec)
+{
+        nv_debug(memx->ppwr, "    DELAY = %d ns\n", nsec);
+        memx_cmd(memx, MEMX_DELAY, 1, (u32[]){ nsec });
+        memx_out(memx); /* fuc can't handle multiple */
+}
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/nv108.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/nv108.c
new file mode 100644
index 000000000000..52c85414866a
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/nv108.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/pwr.h>
+
+#include "fuc/nv108.fuc.h"
+
+struct nv108_pwr_priv {
+        struct nouveau_pwr base;
+};
+
+static int
+nv108_pwr_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+               struct nouveau_oclass *oclass, void *data, u32 size,
+               struct nouveau_object **pobject)
+{
+        struct nv108_pwr_priv *priv;
+        int ret;
+
+        ret = nouveau_pwr_create(parent, engine, oclass, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        priv->base.code.data = nv108_pwr_code;
+        priv->base.code.size = sizeof(nv108_pwr_code);
+        priv->base.data.data = nv108_pwr_data;
+        priv->base.data.size = sizeof(nv108_pwr_data);
+        return 0;
+}
+
+struct nouveau_oclass
+nv108_pwr_oclass = {
+        .handle = NV_SUBDEV(PWR, 0x00),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv108_pwr_ctor,
+                .dtor = _nouveau_pwr_dtor,
+                .init = _nouveau_pwr_init,
+                .fini = _nouveau_pwr_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/nva3.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/nva3.c
new file mode 100644
index 000000000000..c132b7ca9747
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/nva3.c
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/pwr.h>
+
+#include "fuc/nva3.fuc.h"
+
+struct nva3_pwr_priv {
+        struct nouveau_pwr base;
+};
+
+static int
+nva3_pwr_init(struct nouveau_object *object)
+{
+        struct nva3_pwr_priv *priv = (void *)object;
+        nv_mask(priv, 0x022210, 0x00000001, 0x00000000);
+        nv_mask(priv, 0x022210, 0x00000001, 0x00000001);
+        return nouveau_pwr_init(&priv->base);
+}
+
+static int
+nva3_pwr_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+              struct nouveau_oclass *oclass, void *data, u32 size,
+              struct nouveau_object **pobject)
+{
+        struct nva3_pwr_priv *priv;
+        int ret;
+
+        ret = nouveau_pwr_create(parent, engine, oclass, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        priv->base.code.data = nva3_pwr_code;
+        priv->base.code.size = sizeof(nva3_pwr_code);
+        priv->base.data.data = nva3_pwr_data;
+        priv->base.data.size = sizeof(nva3_pwr_data);
+        return 0;
+}
+
+struct nouveau_oclass
+nva3_pwr_oclass = {
+        .handle = NV_SUBDEV(PWR, 0xa3),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nva3_pwr_ctor,
+                .dtor = _nouveau_pwr_dtor,
+                .init = nva3_pwr_init,
+                .fini = _nouveau_pwr_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/nvc0.c
new file mode 100644
index 000000000000..495f6857428d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/nvc0.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/pwr.h>
+
+#include "fuc/nvc0.fuc.h"
+
+struct nvc0_pwr_priv {
+        struct nouveau_pwr base;
+};
+
+static int
+nvc0_pwr_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+              struct nouveau_oclass *oclass, void *data, u32 size,
+              struct nouveau_object **pobject)
+{
+        struct nvc0_pwr_priv *priv;
+        int ret;
+
+        ret = nouveau_pwr_create(parent, engine, oclass, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        priv->base.code.data = nvc0_pwr_code;
+        priv->base.code.size = sizeof(nvc0_pwr_code);
+        priv->base.data.data = nvc0_pwr_data;
+        priv->base.data.size = sizeof(nvc0_pwr_data);
+        return 0;
+}
+
+struct nouveau_oclass
+nvc0_pwr_oclass = {
+        .handle = NV_SUBDEV(PWR, 0xc0),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nvc0_pwr_ctor,
+                .dtor = _nouveau_pwr_dtor,
+                .init = _nouveau_pwr_init,
+                .fini = _nouveau_pwr_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/pwr/nvd0.c b/drivers/gpu/drm/nouveau/core/subdev/pwr/nvd0.c
new file mode 100644
index 000000000000..043aa142fe82
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/pwr/nvd0.c
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/pwr.h>
+
+#include "fuc/nvd0.fuc.h"
+
+struct nvd0_pwr_priv {
+        struct nouveau_pwr base;
+};
+
+static int
+nvd0_pwr_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+              struct nouveau_oclass *oclass, void *data, u32 size,
+              struct nouveau_object **pobject)
+{
+        struct nvd0_pwr_priv *priv;
+        int ret;
+
+        ret = nouveau_pwr_create(parent, engine, oclass, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        priv->base.code.data = nvd0_pwr_code;
+        priv->base.code.size = sizeof(nvd0_pwr_code);
+        priv->base.data.data = nvd0_pwr_data;
+        priv->base.data.size = sizeof(nvd0_pwr_data);
+        return 0;
+}
+
+struct nouveau_oclass
+nvd0_pwr_oclass = {
+        .handle = NV_SUBDEV(PWR, 0xd0),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nvd0_pwr_ctor,
+                .dtor = _nouveau_pwr_dtor,
+                .init = _nouveau_pwr_init,
+                .fini = _nouveau_pwr_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/base.c b/drivers/gpu/drm/nouveau/core/subdev/therm/base.c
index f1de7a9c572b..21b2b3021fad 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/base.c
@@ -92,10 +92,11 @@ nouveau_therm_update(struct nouveau_therm *therm, int mode)
         struct nouveau_timer *ptimer = nouveau_timer(therm);
         struct nouveau_therm_priv *priv = (void *)therm;
         unsigned long flags;
-        int duty;
+        bool immd = true;
+        bool poll = true;
+        int duty = -1;
 
         spin_lock_irqsave(&priv->lock, flags);
-        nv_debug(therm, "FAN speed check\n");
         if (mode < 0)
                 mode = priv->mode;
         priv->mode = mode;
@@ -106,28 +107,48 @@ nouveau_therm_update(struct nouveau_therm *therm, int mode)
                 duty = nouveau_therm_fan_get(therm);
                 if (duty < 0)
                         duty = 100;
+                poll = false;
                 break;
         case NOUVEAU_THERM_CTRL_AUTO:
-                if (priv->fan->bios.nr_fan_trip)
+                if (priv->fan->bios.nr_fan_trip) {
                         duty = nouveau_therm_update_trip(therm);
-                else
+                } else
+                if (priv->fan->bios.linear_min_temp ||
+                    priv->fan->bios.linear_max_temp) {
                         duty = nouveau_therm_update_linear(therm);
+                } else {
+                        duty = priv->cstate;
+                        poll = false;
+                }
+                immd = false;
                 break;
         case NOUVEAU_THERM_CTRL_NONE:
         default:
                 ptimer->alarm_cancel(ptimer, &priv->alarm);
-                goto done;
+                poll = false;
         }
 
-        nv_debug(therm, "FAN target request: %d%%\n", duty);
-        nouveau_therm_fan_set(therm, (mode != NOUVEAU_THERM_CTRL_AUTO), duty);
-
-done:
-        if (list_empty(&priv->alarm.head) && (mode == NOUVEAU_THERM_CTRL_AUTO))
+        if (list_empty(&priv->alarm.head) && poll)
                 ptimer->alarm(ptimer, 1000000000ULL, &priv->alarm);
-        else if (!list_empty(&priv->alarm.head))
-                nv_debug(therm, "therm fan alarm list is not empty\n");
         spin_unlock_irqrestore(&priv->lock, flags);
+
+        if (duty >= 0) {
+                nv_debug(therm, "FAN target request: %d%%\n", duty);
+                nouveau_therm_fan_set(therm, immd, duty);
+        }
+}
+
+int
+nouveau_therm_cstate(struct nouveau_therm *ptherm, int fan, int dir)
+{
+        struct nouveau_therm_priv *priv = (void *)ptherm;
+        if (!dir || (dir < 0 && fan < priv->cstate) ||
+                    (dir > 0 && fan > priv->cstate)) {
+                nv_debug(ptherm, "default fan speed -> %d%%\n", fan);
+                priv->cstate = fan;
+                nouveau_therm_update(ptherm, -1);
+        }
+        return 0;
 }
 
 static void
@@ -149,14 +170,15 @@ nouveau_therm_fan_mode(struct nouveau_therm *therm, int mode)
                 "automatic"
         };
 
-        /* The default PDAEMON ucode interferes with fan management */
+        /* The default PPWR ucode on fermi interferes with fan management */
         if ((mode >= ARRAY_SIZE(name)) ||
-            (mode != NOUVEAU_THERM_CTRL_NONE && device->card_type >= NV_C0))
+            (mode != NOUVEAU_THERM_CTRL_NONE && device->card_type >= NV_C0 &&
+             !nouveau_subdev(device, NVDEV_SUBDEV_PWR)))
                 return -EINVAL;
 
         /* do not allow automatic fan management if the thermal sensor is
          * not available */
-        if (priv->mode == 2 && therm->temp_get(therm) < 0)
+        if (priv->mode == NOUVEAU_THERM_CTRL_AUTO && therm->temp_get(therm) < 0)
                 return -EINVAL;
 
         if (priv->mode == mode)
@@ -335,7 +357,7 @@ nouveau_therm_preinit(struct nouveau_therm *therm)
         nouveau_therm_ic_ctor(therm);
         nouveau_therm_fan_ctor(therm);
 
-        nouveau_therm_fan_mode(therm, NOUVEAU_THERM_CTRL_NONE);
+        nouveau_therm_fan_mode(therm, NOUVEAU_THERM_CTRL_AUTO);
         nouveau_therm_sensor_preinit(therm);
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/fan.c b/drivers/gpu/drm/nouveau/core/subdev/therm/fan.c
index 39f47b950ad1..95f6129eeede 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/fan.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/fan.c
@@ -185,8 +185,11 @@ nouveau_therm_fan_set_defaults(struct nouveau_therm *therm)
         priv->fan->bios.max_duty = 100;
         priv->fan->bios.bump_period = 500;
         priv->fan->bios.slow_down_period = 2000;
+/*XXX: talk to mupuf */
+#if 0
         priv->fan->bios.linear_min_temp = 40;
         priv->fan->bios.linear_max_temp = 85;
+#endif
 }
 
 static void
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/fantog.c b/drivers/gpu/drm/nouveau/core/subdev/therm/fantog.c
index e601773ee475..f69dab11f720 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/fantog.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/fantog.c
@@ -97,6 +97,13 @@ nouveau_fantog_create(struct nouveau_therm *therm, struct dcb_gpio_func *func)
 {
         struct nouveau_therm_priv *tpriv = (void *)therm;
         struct nouveau_fantog_priv *priv;
+        int ret;
+
+        if (therm->pwm_ctrl) {
+                ret = therm->pwm_ctrl(therm, func->line, false);
+                if (ret)
+                        return ret;
+        }
 
         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
         tpriv->fan = &priv->base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/ic.c b/drivers/gpu/drm/nouveau/core/subdev/therm/ic.c
index 8b3adec5fbb1..13b850076443 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/ic.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/ic.c
@@ -55,28 +55,28 @@ probe_monitoring_device(struct nouveau_i2c_port *i2c,
         return true;
 }
 
-static struct i2c_board_info
+static struct nouveau_i2c_board_info
 nv_board_infos[] = {
-        { I2C_BOARD_INFO("w83l785ts", 0x2d) },
-        { I2C_BOARD_INFO("w83781d", 0x2d) },
-        { I2C_BOARD_INFO("adt7473", 0x2e) },
-        { I2C_BOARD_INFO("adt7473", 0x2d) },
-        { I2C_BOARD_INFO("adt7473", 0x2c) },
-        { I2C_BOARD_INFO("f75375", 0x2e) },
-        { I2C_BOARD_INFO("lm99", 0x4c) },
-        { I2C_BOARD_INFO("lm90", 0x4c) },
-        { I2C_BOARD_INFO("lm90", 0x4d) },
-        { I2C_BOARD_INFO("adm1021", 0x18) },
-        { I2C_BOARD_INFO("adm1021", 0x19) },
-        { I2C_BOARD_INFO("adm1021", 0x1a) },
-        { I2C_BOARD_INFO("adm1021", 0x29) },
-        { I2C_BOARD_INFO("adm1021", 0x2a) },
-        { I2C_BOARD_INFO("adm1021", 0x2b) },
-        { I2C_BOARD_INFO("adm1021", 0x4c) },
-        { I2C_BOARD_INFO("adm1021", 0x4d) },
-        { I2C_BOARD_INFO("adm1021", 0x4e) },
-        { I2C_BOARD_INFO("lm63", 0x18) },
-        { I2C_BOARD_INFO("lm63", 0x4e) },
+        { { I2C_BOARD_INFO("w83l785ts", 0x2d) }, 0 },
+        { { I2C_BOARD_INFO("w83781d", 0x2d) }, 0  },
+        { { I2C_BOARD_INFO("adt7473", 0x2e) }, 20  },
+        { { I2C_BOARD_INFO("adt7473", 0x2d) }, 20  },
+        { { I2C_BOARD_INFO("adt7473", 0x2c) }, 20  },
+        { { I2C_BOARD_INFO("f75375", 0x2e) }, 0  },
+        { { I2C_BOARD_INFO("lm99", 0x4c) }, 0  },
+        { { I2C_BOARD_INFO("lm90", 0x4c) }, 0  },
+        { { I2C_BOARD_INFO("lm90", 0x4d) }, 0  },
+        { { I2C_BOARD_INFO("adm1021", 0x18) }, 0  },
+        { { I2C_BOARD_INFO("adm1021", 0x19) }, 0  },
+        { { I2C_BOARD_INFO("adm1021", 0x1a) }, 0  },
+        { { I2C_BOARD_INFO("adm1021", 0x29) }, 0  },
+        { { I2C_BOARD_INFO("adm1021", 0x2a) }, 0  },
+        { { I2C_BOARD_INFO("adm1021", 0x2b) }, 0  },
+        { { I2C_BOARD_INFO("adm1021", 0x4c) }, 0  },
+        { { I2C_BOARD_INFO("adm1021", 0x4d) }, 0  },
+        { { I2C_BOARD_INFO("adm1021", 0x4e) }, 0  },
+        { { I2C_BOARD_INFO("lm63", 0x18) }, 0  },
+        { { I2C_BOARD_INFO("lm63", 0x4e) }, 0  },
         { }
 };
 
@@ -89,9 +89,9 @@ nouveau_therm_ic_ctor(struct nouveau_therm *therm)
         struct nvbios_extdev_func extdev_entry;
 
         if (!nvbios_extdev_find(bios, NVBIOS_EXTDEV_LM89, &extdev_entry)) {
-                struct i2c_board_info board[] = {
-                        { I2C_BOARD_INFO("lm90", extdev_entry.addr >> 1) },
-                        { }
+                struct nouveau_i2c_board_info board[] = {
+                  { { I2C_BOARD_INFO("lm90", extdev_entry.addr >> 1) }, 0},
+                  { }
                 };
 
                 i2c->identify(i2c, NV_I2C_DEFAULT(0), "monitoring device",
@@ -101,9 +101,9 @@ nouveau_therm_ic_ctor(struct nouveau_therm *therm)
         }
 
         if (!nvbios_extdev_find(bios, NVBIOS_EXTDEV_ADT7473, &extdev_entry)) {
-                struct i2c_board_info board[] = {
-                        { I2C_BOARD_INFO("adt7473", extdev_entry.addr >> 1) },
-                        { }
+                struct nouveau_i2c_board_info board[] = {
+                  { { I2C_BOARD_INFO("adt7473", extdev_entry.addr >> 1) }, 20 },
+                  { }
                 };
 
                 i2c->identify(i2c, NV_I2C_DEFAULT(0), "monitoring device",
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/nv84.c b/drivers/gpu/drm/nouveau/core/subdev/therm/nv84.c
index 42ba633ccff7..1d15c52fad0c 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/nv84.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/nv84.c
@@ -126,7 +126,7 @@ nv84_therm_intr(struct nouveau_subdev *subdev)
 
         spin_lock_irqsave(&priv->sensor.alarm_program_lock, flags);
 
-        intr = nv_rd32(therm, 0x20100);
+        intr = nv_rd32(therm, 0x20100) & 0x3ff;
 
         /* THRS_4: downclock */
         if (intr & 0x002) {
@@ -209,6 +209,19 @@ nv84_therm_ctor(struct nouveau_object *parent,
         return nouveau_therm_preinit(&priv->base.base);
 }
 
+int
+nv84_therm_fini(struct nouveau_object *object, bool suspend)
+{
+        /* Disable PTherm IRQs */
+        nv_wr32(object, 0x20000, 0x00000000);
+
+        /* ACK all PTherm IRQs */
+        nv_wr32(object, 0x20100, 0xffffffff);
+        nv_wr32(object, 0x1100, 0x10000); /* PBUS */
+
+        return _nouveau_therm_fini(object, suspend);
+}
+
 struct nouveau_oclass
 nv84_therm_oclass = {
         .handle = NV_SUBDEV(THERM, 0x84),
@@ -216,6 +229,6 @@ nv84_therm_oclass = {
                 .ctor = nv84_therm_ctor,
                 .dtor = _nouveau_therm_dtor,
                 .init = _nouveau_therm_init,
-                .fini = _nouveau_therm_fini,
+                .fini = nv84_therm_fini,
         },
 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/nva3.c b/drivers/gpu/drm/nouveau/core/subdev/therm/nva3.c
index d11a7c400813..3b2c4580098b 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/nva3.c
@@ -94,6 +94,6 @@ nva3_therm_oclass = {
                 .ctor = nva3_therm_ctor,
                 .dtor = _nouveau_therm_dtor,
                 .init = nva3_therm_init,
-                .fini = _nouveau_therm_fini,
+                .fini = nv84_therm_fini,
         },
 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/nvd0.c b/drivers/gpu/drm/nouveau/core/subdev/therm/nvd0.c
index 54c28bdc4204..4dd4f81ae873 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/nvd0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/nvd0.c
@@ -148,6 +148,6 @@ nvd0_therm_oclass = {
                 .ctor = nvd0_therm_ctor,
                 .dtor = _nouveau_therm_dtor,
                 .init = nvd0_therm_init,
-                .fini = _nouveau_therm_fini,
+                .fini = nv84_therm_fini,
         },
 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/priv.h b/drivers/gpu/drm/nouveau/core/subdev/therm/priv.h
index dd38529262fb..96f8f95693ce 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/priv.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/priv.h
@@ -76,6 +76,7 @@ struct nouveau_therm_priv {
         spinlock_t lock;
         struct nouveau_therm_trip_point *last_trip;
         int mode;
+        int cstate;
         int suspend;
 
         /* bios */
@@ -144,6 +145,7 @@ int nv50_fan_pwm_get(struct nouveau_therm *, int, u32 *, u32 *);
 int nv50_fan_pwm_set(struct nouveau_therm *, int, u32, u32);
 int nv50_fan_pwm_clock(struct nouveau_therm *);
 int nv84_temp_get(struct nouveau_therm *therm);
+int nv84_therm_fini(struct nouveau_object *object, bool suspend);
 
 int nva3_therm_fan_sense(struct nouveau_therm *);
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/temp.c b/drivers/gpu/drm/nouveau/core/subdev/therm/temp.c
index b80a33011b93..cfde9eb44ad0 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/temp.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/temp.c
@@ -180,8 +180,6 @@ alarm_timer_callback(struct nouveau_alarm *alarm)
 
         spin_lock_irqsave(&priv->sensor.alarm_program_lock, flags);
 
-        nv_debug(therm, "polling the internal temperature\n");
-
         nouveau_therm_threshold_hyst_polling(therm, &sensor->thrs_fan_boost,
                                              NOUVEAU_THERM_THRS_FANBOOST);
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/timer/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/timer/nv04.c
index 57711ecb566c..c0bdd10358d7 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/timer/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/timer/nv04.c
@@ -119,16 +119,8 @@ nv04_timer_alarm_cancel(struct nouveau_timer *ptimer,
 {
         struct nv04_timer_priv *priv = (void *)ptimer;
         unsigned long flags;
-
-        /* avoid deleting an entry while the alarm intr is running */
         spin_lock_irqsave(&priv->lock, flags);
-
-        /* delete the alarm from the list */
-        list_del(&alarm->head);
-
-        /* reset the head so as list_empty returns 1 */
-        INIT_LIST_HEAD(&alarm->head);
-
+        list_del_init(&alarm->head);
         spin_unlock_irqrestore(&priv->lock, flags);
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/volt/base.c b/drivers/gpu/drm/nouveau/core/subdev/volt/base.c
new file mode 100644
index 000000000000..32794a999106
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/volt/base.c
@@ -0,0 +1,198 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/volt.h>
+
+#include <subdev/bios.h>
+#include <subdev/bios/vmap.h>
+#include <subdev/bios/volt.h>
+
+static int
+nouveau_volt_get(struct nouveau_volt *volt)
+{
+        if (volt->vid_get) {
+                int ret = volt->vid_get(volt), i;
+                if (ret >= 0) {
+                        for (i = 0; i < volt->vid_nr; i++) {
+                                if (volt->vid[i].vid == ret)
+                                        return volt->vid[i].uv;
+                        }
+                        ret = -EINVAL;
+                }
+                return ret;
+        }
+        return -ENODEV;
+}
+
+static int
+nouveau_volt_set(struct nouveau_volt *volt, u32 uv)
+{
+        if (volt->vid_set) {
+                int i, ret = -EINVAL;
+                for (i = 0; i < volt->vid_nr; i++) {
+                        if (volt->vid[i].uv == uv) {
+                                ret = volt->vid_set(volt, volt->vid[i].vid);
+                                nv_debug(volt, "set %duv: %d\n", uv, ret);
+                                break;
+                        }
+                }
+                return ret;
+        }
+        return -ENODEV;
+}
+
+static int
+nouveau_volt_map(struct nouveau_volt *volt, u8 id)
+{
+        struct nouveau_bios *bios = nouveau_bios(volt);
+        struct nvbios_vmap_entry info;
+        u8  ver, len;
+        u16 vmap;
+
+        vmap = nvbios_vmap_entry_parse(bios, id, &ver, &len, &info);
+        if (vmap) {
+                if (info.link != 0xff) {
+                        int ret = nouveau_volt_map(volt, info.link);
+                        if (ret < 0)
+                                return ret;
+                        info.min += ret;
+                }
+                return info.min;
+        }
+
+        return id ? id * 10000 : -ENODEV;
+}
+
+static int
+nouveau_volt_set_id(struct nouveau_volt *volt, u8 id, int condition)
+{
+        int ret = nouveau_volt_map(volt, id);
+        if (ret >= 0) {
+                int prev = nouveau_volt_get(volt);
+                if (!condition || prev < 0 ||
+                    (condition < 0 && ret < prev) ||
+                    (condition > 0 && ret > prev)) {
+                        ret = nouveau_volt_set(volt, ret);
+                } else {
+                        ret = 0;
+                }
+        }
+        return ret;
+}
+
+int
+_nouveau_volt_init(struct nouveau_object *object)
+{
+        struct nouveau_volt *volt = (void *)object;
+        int ret;
+
+        ret = nouveau_subdev_init(&volt->base);
+        if (ret)
+                return ret;
+
+        ret = volt->get(volt);
+        if (ret < 0) {
+                if (ret != -ENODEV)
+                        nv_debug(volt, "current voltage unknown\n");
+                return 0;
+        }
+
+        nv_info(volt, "GPU voltage: %duv\n", ret);
+        return 0;
+}
+
+void
+_nouveau_volt_dtor(struct nouveau_object *object)
+{
+        struct nouveau_volt *volt = (void *)object;
+        nouveau_subdev_destroy(&volt->base);
+}
+
+int
+nouveau_volt_create_(struct nouveau_object *parent,
+                     struct nouveau_object *engine,
+                     struct nouveau_oclass *oclass, int length, void **pobject)
+{
+        struct nouveau_bios *bios = nouveau_bios(parent);
+        struct nouveau_volt *volt;
+        struct nvbios_volt_entry ivid;
+        struct nvbios_volt info;
+        u8  ver, hdr, cnt, len;
+        u16 data;
+        int ret, i;
+
+        ret = nouveau_subdev_create_(parent, engine, oclass, 0, "VOLT",
+                                     "voltage", length, pobject);
+        volt = *pobject;
+        if (ret)
+                return ret;
+
+        volt->get = nouveau_volt_get;
+        volt->set = nouveau_volt_set;
+        volt->set_id = nouveau_volt_set_id;
+
+        data = nvbios_volt_parse(bios, &ver, &hdr, &cnt, &len, &info);
+        if (data && info.vidmask && info.base && info.step) {
+                for (i = 0; i < info.vidmask + 1; i++) {
+                        if (info.base >= info.min &&
+                            info.base <= info.max) {
+                                volt->vid[volt->vid_nr].uv = info.base;
+                                volt->vid[volt->vid_nr].vid = i;
+                                volt->vid_nr++;
+                        }
+                        info.base += info.step;
+                }
+                volt->vid_mask = info.vidmask;
+        } else
+        if (data && info.vidmask) {
+                for (i = 0; i < cnt; i++) {
+                        data = nvbios_volt_entry_parse(bios, i, &ver, &hdr,
+                                                      &ivid);
+                        if (data) {
+                                volt->vid[volt->vid_nr].uv = ivid.voltage;
+                                volt->vid[volt->vid_nr].vid = ivid.vid;
+                                volt->vid_nr++;
+                        }
+                }
+                volt->vid_mask = info.vidmask;
+        }
+
+        if (volt->vid_nr) {
+                for (i = 0; i < volt->vid_nr; i++) {
+                        nv_debug(volt, "VID %02x: %duv\n",
+                                 volt->vid[i].vid, volt->vid[i].uv);
+                }
+
+                /*XXX: this is an assumption.. there probably exists boards
+                 * out there with i2c-connected voltage controllers too..
+                 */
+                ret = nouveau_voltgpio_init(volt);
+                if (ret == 0) {
+                        volt->vid_get = nouveau_voltgpio_get;
+                        volt->vid_set = nouveau_voltgpio_set;
+                }
+        }
+
+        return ret;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/volt/gpio.c b/drivers/gpu/drm/nouveau/core/subdev/volt/gpio.c
new file mode 100644
index 000000000000..755fa91bcd09
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/volt/gpio.c
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/volt.h>
+#include <subdev/gpio.h>
+#include <subdev/bios/gpio.h>
+
+static const u8 tags[] = {
+        DCB_GPIO_VID0, DCB_GPIO_VID1, DCB_GPIO_VID2, DCB_GPIO_VID3,
+        DCB_GPIO_VID4, DCB_GPIO_VID5, DCB_GPIO_VID6, DCB_GPIO_VID7,
+};
+
+int
+nouveau_voltgpio_get(struct nouveau_volt *volt)
+{
+        struct nouveau_gpio *gpio = nouveau_gpio(volt);
+        u8 vid = 0;
+        int i;
+
+        for (i = 0; i < ARRAY_SIZE(tags); i++) {
+                if (volt->vid_mask & (1 << i)) {
+                        int ret = gpio->get(gpio, 0, tags[i], 0xff);
+                        if (ret < 0)
+                                return ret;
+                        vid |= ret << i;
+                }
+        }
+
+        return vid;
+}
+
+int
+nouveau_voltgpio_set(struct nouveau_volt *volt, u8 vid)
+{
+        struct nouveau_gpio *gpio = nouveau_gpio(volt);
+        int i;
+
+        for (i = 0; i < ARRAY_SIZE(tags); i++, vid >>= 1) {
+                if (volt->vid_mask & (1 << i)) {
+                        int ret = gpio->set(gpio, 0, tags[i], 0xff, vid & 1);
+                        if (ret < 0)
+                                return ret;
+                }
+        }
+
+        return 0;
+}
+
+int
+nouveau_voltgpio_init(struct nouveau_volt *volt)
+{
+        struct nouveau_gpio *gpio = nouveau_gpio(volt);
+        struct dcb_gpio_func func;
+        int i;
+
+        /* check we have gpio function info for each vid bit.  on some
+         * boards (ie. nvs295) the vid mask has more bits than there
+         * are valid gpio functions... from traces, nvidia appear to
+         * just touch the existing ones, so let's mask off the invalid
+         * bits and continue with life
+         */
+        for (i = 0; i < ARRAY_SIZE(tags); i++) {
+                if (volt->vid_mask & (1 << i)) {
+                        int ret = gpio->find(gpio, 0, tags[i], 0xff, &func);
+                        if (ret) {
+                                if (ret != -ENOENT)
+                                        return ret;
+                                nv_debug(volt, "VID bit %d has no GPIO\n", i);
+                                volt->vid_mask &= ~(1 << i);
+                        }
+                }
+        }
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/volt/nv40.c b/drivers/gpu/drm/nouveau/core/subdev/volt/nv40.c
new file mode 100644
index 000000000000..87d5358376a6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/volt/nv40.c
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/volt.h>
+
+struct nv40_volt_priv {
+        struct nouveau_volt base;
+};
+
+static int
+nv40_volt_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+               struct nouveau_oclass *oclass, void *data, u32 size,
+               struct nouveau_object **pobject)
+{
+        struct nv40_volt_priv *priv;
+        int ret;
+
+        ret = nouveau_volt_create(parent, engine, oclass, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        return 0;
+}
+
+struct nouveau_oclass
+nv40_volt_oclass = {
+        .handle = NV_SUBDEV(VOLT, 0x40),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv40_volt_ctor,
+                .dtor = _nouveau_volt_dtor,
+                .init = _nouveau_volt_init,
+                .fini = _nouveau_volt_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/dispnv04/Makefile b/drivers/gpu/drm/nouveau/dispnv04/Makefile
index ea3f5b8a0f95..424a489d0f03 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/Makefile
+++ b/drivers/gpu/drm/nouveau/dispnv04/Makefile
@@ -5,6 +5,7 @@ nouveau-y += dispnv04/dac.o
 nouveau-y += dispnv04/dfp.o
 nouveau-y += dispnv04/disp.o
 nouveau-y += dispnv04/hw.o
+nouveau-y += dispnv04/overlay.o
 nouveau-y += dispnv04/tvmodesnv17.o
 nouveau-y += dispnv04/tvnv04.o
 nouveau-y += dispnv04/tvnv17.o
diff --git a/drivers/gpu/drm/nouveau/dispnv04/dfp.c b/drivers/gpu/drm/nouveau/dispnv04/dfp.c
index 59d1c040b84f..936a71c59080 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/dfp.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/dfp.c
@@ -493,7 +493,7 @@ static void nv04_dfp_update_backlight(struct drm_encoder *encoder, int mode)
         if (dev->pdev->device == 0x0174 || dev->pdev->device == 0x0179 ||
             dev->pdev->device == 0x0189 || dev->pdev->device == 0x0329) {
                 if (mode == DRM_MODE_DPMS_ON) {
-                        nv_mask(device, NV_PBUS_DEBUG_DUALHEAD_CTL, 0, 1 << 31);
+                        nv_mask(device, NV_PBUS_DEBUG_DUALHEAD_CTL, 1 << 31, 1 << 31);
                         nv_mask(device, NV_PCRTC_GPIO_EXT, 3, 1);
                 } else {
                         nv_mask(device, NV_PBUS_DEBUG_DUALHEAD_CTL, 1 << 31, 0);
@@ -625,13 +625,15 @@ static void nv04_tmds_slave_init(struct drm_encoder *encoder)
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_i2c *i2c = nouveau_i2c(drm->device);
         struct nouveau_i2c_port *port = i2c->find(i2c, 2);
-        struct i2c_board_info info[] = {
+        struct nouveau_i2c_board_info info[] = {
                 {
-                        .type = "sil164",
-                        .addr = (dcb->tmdsconf.slave_addr == 0x7 ? 0x3a : 0x38),
-                        .platform_data = &(struct sil164_encoder_params) {
-                                SIL164_INPUT_EDGE_RISING
-                        }
+                    {
+                        .type = "sil164",
+                        .addr = (dcb->tmdsconf.slave_addr == 0x7 ? 0x3a : 0x38),
+                        .platform_data = &(struct sil164_encoder_params) {
+                            SIL164_INPUT_EDGE_RISING
+                         }
+                    }, 0
                 },
                 { }
         };
@@ -646,7 +648,7 @@ static void nv04_tmds_slave_init(struct drm_encoder *encoder)
                 return;
 
         drm_i2c_encoder_init(dev, to_encoder_slave(encoder),
-                             &port->adapter, &info[type]);
+                             &port->adapter, &info[type].dev);
 }
 
 static const struct drm_encoder_helper_funcs nv04_lvds_helper_funcs = {
diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.c b/drivers/gpu/drm/nouveau/dispnv04/disp.c
index 4908d3fd0486..b13ff0fc42de 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/disp.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/disp.c
@@ -140,6 +140,8 @@ nv04_display_create(struct drm_device *dev)
                 func->save(encoder);
         }
 
+        nouveau_overlay_init(dev);
+
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.h b/drivers/gpu/drm/nouveau/dispnv04/disp.h
index 2cf65e0b517e..56a28db04000 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/disp.h
+++ b/drivers/gpu/drm/nouveau/dispnv04/disp.h
@@ -123,6 +123,9 @@ int nv04_tv_create(struct drm_connector *, struct dcb_output *);
 /* nv17_tv.c */
 int nv17_tv_create(struct drm_connector *, struct dcb_output *);
 
+/* overlay.c */
+void nouveau_overlay_init(struct drm_device *dev);
+
 static inline bool
 nv_two_heads(struct drm_device *dev)
 {
diff --git a/drivers/gpu/drm/nouveau/dispnv04/hw.c b/drivers/gpu/drm/nouveau/dispnv04/hw.c
index f8dee834527f..aca76af115b3 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/hw.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/hw.c
@@ -27,6 +27,7 @@
 #include "hw.h"
 
 #include <subdev/bios/pll.h>
+#include <subdev/fb.h>
 #include <subdev/clock.h>
 #include <subdev/timer.h>
 
@@ -664,6 +665,7 @@ nv_load_state_ext(struct drm_device *dev, int head,
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_device *device = nv_device(drm->device);
         struct nouveau_timer *ptimer = nouveau_timer(device);
+        struct nouveau_fb *pfb = nouveau_fb(device);
         struct nv04_crtc_reg *regp = &state->crtc_reg[head];
         uint32_t reg900;
         int i;
@@ -680,10 +682,10 @@ nv_load_state_ext(struct drm_device *dev, int head,
                 nv_wr32(device, NV_PVIDEO_INTR_EN, 0);
                 nv_wr32(device, NV_PVIDEO_OFFSET_BUFF(0), 0);
                 nv_wr32(device, NV_PVIDEO_OFFSET_BUFF(1), 0);
-                nv_wr32(device, NV_PVIDEO_LIMIT(0), 0); //drm->fb_available_size - 1);
-                nv_wr32(device, NV_PVIDEO_LIMIT(1), 0); //drm->fb_available_size - 1);
-                nv_wr32(device, NV_PVIDEO_UVPLANE_LIMIT(0), 0); //drm->fb_available_size - 1);
-                nv_wr32(device, NV_PVIDEO_UVPLANE_LIMIT(1), 0); //drm->fb_available_size - 1);
+                nv_wr32(device, NV_PVIDEO_LIMIT(0), pfb->ram->size - 1);
+                nv_wr32(device, NV_PVIDEO_LIMIT(1), pfb->ram->size - 1);
+                nv_wr32(device, NV_PVIDEO_UVPLANE_LIMIT(0), pfb->ram->size - 1);
+                nv_wr32(device, NV_PVIDEO_UVPLANE_LIMIT(1), pfb->ram->size - 1);
                 nv_wr32(device, NV_PBUS_POWERCTRL_2, 0);
 
                 NVWriteCRTC(dev, head, NV_PCRTC_CURSOR_CONFIG, regp->cursor_cfg);
@@ -740,7 +742,7 @@ nv_load_state_ext(struct drm_device *dev, int head,
         }
         /* NV11 and NV20 stop at 0x52. */
         if (nv_gf4_disp_arch(dev)) {
-                if (nv_device(drm->device)->card_type == NV_10) {
+                if (nv_device(drm->device)->card_type < NV_20) {
                         /* Not waiting for vertical retrace before modifying
                            CRE_53/CRE_54 causes lockups. */
                         nouveau_timer_wait_eq(ptimer, 650000000, NV_PRMCIO_INP0__COLOR, 0x8, 0x8);
diff --git a/drivers/gpu/drm/nouveau/dispnv04/overlay.c b/drivers/gpu/drm/nouveau/dispnv04/overlay.c
new file mode 100644
index 000000000000..3618ac6b6316
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/dispnv04/overlay.c
@@ -0,0 +1,320 @@
+/*
+ * Copyright 2013 Ilia Mirkin
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+ * OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Implementation based on the pre-KMS implementation in xf86-video-nouveau,
+ * written by Arthur Huillet.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_fourcc.h>
+
+#include "nouveau_drm.h"
+
+#include "nouveau_bo.h"
+#include "nouveau_connector.h"
+#include "nouveau_display.h"
+#include "nvreg.h"
+
+
+struct nouveau_plane {
+        struct drm_plane base;
+        bool flip;
+        struct nouveau_bo *cur;
+
+        struct {
+                struct drm_property *colorkey;
+                struct drm_property *contrast;
+                struct drm_property *brightness;
+                struct drm_property *hue;
+                struct drm_property *saturation;
+                struct drm_property *iturbt_709;
+        } props;
+
+        int colorkey;
+        int contrast;
+        int brightness;
+        int hue;
+        int saturation;
+        int iturbt_709;
+};
+
+static uint32_t formats[] = {
+        DRM_FORMAT_NV12,
+        DRM_FORMAT_UYVY,
+};
+
+/* Sine can be approximated with
+ * http://en.wikipedia.org/wiki/Bhaskara_I's_sine_approximation_formula
+ * sin(x degrees) ~= 4 x (180 - x) / (40500 - x (180 - x) )
+ * Note that this only works for the range [0, 180].
+ * Also note that sin(x) == -sin(x - 180)
+ */
+static inline int
+sin_mul(int degrees, int factor)
+{
+        if (degrees > 180) {
+                degrees -= 180;
+                factor *= -1;
+        }
+        return factor * 4 * degrees * (180 - degrees) /
+                (40500 - degrees * (180 - degrees));
+}
+
+/* cos(x) = sin(x + 90) */
+static inline int
+cos_mul(int degrees, int factor)
+{
+        return sin_mul((degrees + 90) % 360, factor);
+}
+
+static int
+nv10_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
+                  struct drm_framebuffer *fb, int crtc_x, int crtc_y,
+                  unsigned int crtc_w, unsigned int crtc_h,
+                  uint32_t src_x, uint32_t src_y,
+                  uint32_t src_w, uint32_t src_h)
+{
+        struct nouveau_device *dev = nouveau_dev(plane->dev);
+        struct nouveau_plane *nv_plane = (struct nouveau_plane *)plane;
+        struct nouveau_framebuffer *nv_fb = nouveau_framebuffer(fb);
+        struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+        struct nouveau_bo *cur = nv_plane->cur;
+        bool flip = nv_plane->flip;
+        int format = ALIGN(src_w * 4, 0x100);
+        int soff = NV_PCRTC0_SIZE * nv_crtc->index;
+        int soff2 = NV_PCRTC0_SIZE * !nv_crtc->index;
+        int ret;
+
+        if (format > 0xffff)
+                return -EINVAL;
+
+        ret = nouveau_bo_pin(nv_fb->nvbo, TTM_PL_FLAG_VRAM);
+        if (ret)
+                return ret;
+
+        nv_plane->cur = nv_fb->nvbo;
+
+        /* Source parameters given in 16.16 fixed point, ignore fractional. */
+        src_x = src_x >> 16;
+        src_y = src_y >> 16;
+        src_w = src_w >> 16;
+        src_h = src_h >> 16;
+
+        nv_mask(dev, NV_PCRTC_ENGINE_CTRL + soff, NV_CRTC_FSEL_OVERLAY, NV_CRTC_FSEL_OVERLAY);
+        nv_mask(dev, NV_PCRTC_ENGINE_CTRL + soff2, NV_CRTC_FSEL_OVERLAY, 0);
+
+        nv_wr32(dev, NV_PVIDEO_BASE(flip), 0);
+        nv_wr32(dev, NV_PVIDEO_OFFSET_BUFF(flip), nv_fb->nvbo->bo.offset);
+        nv_wr32(dev, NV_PVIDEO_SIZE_IN(flip), src_h << 16 | src_w);
+        nv_wr32(dev, NV_PVIDEO_POINT_IN(flip), src_y << 16 | src_x);
+        nv_wr32(dev, NV_PVIDEO_DS_DX(flip), (src_w << 20) / crtc_w);
+        nv_wr32(dev, NV_PVIDEO_DT_DY(flip), (src_h << 20) / crtc_h);
+        nv_wr32(dev, NV_PVIDEO_POINT_OUT(flip), crtc_y << 16 | crtc_x);
+        nv_wr32(dev, NV_PVIDEO_SIZE_OUT(flip), crtc_h << 16 | crtc_w);
+
+        if (fb->pixel_format == DRM_FORMAT_NV12) {
+                format |= NV_PVIDEO_FORMAT_COLOR_LE_CR8YB8CB8YA8;
+                format |= NV_PVIDEO_FORMAT_PLANAR;
+        }
+        if (nv_plane->iturbt_709)
+                format |= NV_PVIDEO_FORMAT_MATRIX_ITURBT709;
+        if (nv_plane->colorkey & (1 << 24))
+                format |= NV_PVIDEO_FORMAT_DISPLAY_COLOR_KEY;
+
+        if (fb->pixel_format == DRM_FORMAT_NV12) {
+                nv_wr32(dev, NV_PVIDEO_UVPLANE_BASE(flip), 0);
+                nv_wr32(dev, NV_PVIDEO_UVPLANE_OFFSET_BUFF(flip),
+                        nv_fb->nvbo->bo.offset + fb->offsets[1]);
+        }
+        nv_wr32(dev, NV_PVIDEO_FORMAT(flip), format);
+        nv_wr32(dev, NV_PVIDEO_STOP, 0);
+        /* TODO: wait for vblank? */
+        nv_wr32(dev, NV_PVIDEO_BUFFER, flip ? 0x10 : 0x1);
+        nv_plane->flip = !flip;
+
+        if (cur)
+                nouveau_bo_unpin(cur);
+
+        return 0;
+}
+
+static int
+nv10_disable_plane(struct drm_plane *plane)
+{
+        struct nouveau_device *dev = nouveau_dev(plane->dev);
+        struct nouveau_plane *nv_plane = (struct nouveau_plane *)plane;
+
+        nv_wr32(dev, NV_PVIDEO_STOP, 1);
+        if (nv_plane->cur) {
+                nouveau_bo_unpin(nv_plane->cur);
+                nv_plane->cur = NULL;
+        }
+
+        return 0;
+}
+
+static void
+nv10_destroy_plane(struct drm_plane *plane)
+{
+        nv10_disable_plane(plane);
+        drm_plane_cleanup(plane);
+        kfree(plane);
+}
+
+static void
+nv10_set_params(struct nouveau_plane *plane)
+{
+        struct nouveau_device *dev = nouveau_dev(plane->base.dev);
+        u32 luma = (plane->brightness - 512) << 16 | plane->contrast;
+        u32 chroma = ((sin_mul(plane->hue, plane->saturation) & 0xffff) << 16) |
+                (cos_mul(plane->hue, plane->saturation) & 0xffff);
+        u32 format = 0;
+
+        nv_wr32(dev, NV_PVIDEO_LUMINANCE(0), luma);
+        nv_wr32(dev, NV_PVIDEO_LUMINANCE(1), luma);
+        nv_wr32(dev, NV_PVIDEO_CHROMINANCE(0), chroma);
+        nv_wr32(dev, NV_PVIDEO_CHROMINANCE(1), chroma);
+        nv_wr32(dev, NV_PVIDEO_COLOR_KEY, plane->colorkey & 0xffffff);
+
+        if (plane->cur) {
+                if (plane->iturbt_709)
+                        format |= NV_PVIDEO_FORMAT_MATRIX_ITURBT709;
+                if (plane->colorkey & (1 << 24))
+                        format |= NV_PVIDEO_FORMAT_DISPLAY_COLOR_KEY;
+                nv_mask(dev, NV_PVIDEO_FORMAT(plane->flip),
+                        NV_PVIDEO_FORMAT_MATRIX_ITURBT709 |
+                        NV_PVIDEO_FORMAT_DISPLAY_COLOR_KEY,
+                        format);
+        }
+}
+
+static int
+nv10_set_property(struct drm_plane *plane,
+                  struct drm_property *property,
+                  uint64_t value)
+{
+        struct nouveau_plane *nv_plane = (struct nouveau_plane *)plane;
+
+        if (property == nv_plane->props.colorkey)
+                nv_plane->colorkey = value;
+        else if (property == nv_plane->props.contrast)
+                nv_plane->contrast = value;
+        else if (property == nv_plane->props.brightness)
+                nv_plane->brightness = value;
+        else if (property == nv_plane->props.hue)
+                nv_plane->hue = value;
+        else if (property == nv_plane->props.saturation)
+                nv_plane->saturation = value;
+        else if (property == nv_plane->props.iturbt_709)
+                nv_plane->iturbt_709 = value;
+        else
+                return -EINVAL;
+
+        nv10_set_params(nv_plane);
+        return 0;
+}
+
+static const struct drm_plane_funcs nv10_plane_funcs = {
+        .update_plane = nv10_update_plane,
+        .disable_plane = nv10_disable_plane,
+        .set_property = nv10_set_property,
+        .destroy = nv10_destroy_plane,
+};
+
+static void
+nv10_overlay_init(struct drm_device *device)
+{
+        struct nouveau_device *dev = nouveau_dev(device);
+        struct nouveau_plane *plane = kzalloc(sizeof(struct nouveau_plane), GFP_KERNEL);
+        int ret;
+
+        if (!plane)
+                return;
+
+        ret = drm_plane_init(device, &plane->base, 3 /* both crtc's */,
+                             &nv10_plane_funcs,
+                             formats, ARRAY_SIZE(formats), false);
+        if (ret)
+                goto err;
+
+        /* Set up the plane properties */
+        plane->props.colorkey = drm_property_create_range(
+                        device, 0, "colorkey", 0, 0x01ffffff);
+        plane->props.contrast = drm_property_create_range(
+                        device, 0, "contrast", 0, 8192 - 1);
+        plane->props.brightness = drm_property_create_range(
+                        device, 0, "brightness", 0, 1024);
+        plane->props.hue = drm_property_create_range(
+                        device, 0, "hue", 0, 359);
+        plane->props.saturation = drm_property_create_range(
+                        device, 0, "saturation", 0, 8192 - 1);
+        plane->props.iturbt_709 = drm_property_create_range(
+                        device, 0, "iturbt_709", 0, 1);
+        if (!plane->props.colorkey ||
+            !plane->props.contrast ||
+            !plane->props.brightness ||
+            !plane->props.hue ||
+            !plane->props.saturation ||
+            !plane->props.iturbt_709)
+                goto cleanup;
+
+        plane->colorkey = 0;
+        drm_object_attach_property(&plane->base.base,
+                                   plane->props.colorkey, plane->colorkey);
+
+        plane->contrast = 0x1000;
+        drm_object_attach_property(&plane->base.base,
+                                   plane->props.contrast, plane->contrast);
+
+        plane->brightness = 512;
+        drm_object_attach_property(&plane->base.base,
+                                   plane->props.brightness, plane->brightness);
+
+        plane->hue = 0;
+        drm_object_attach_property(&plane->base.base,
+                                   plane->props.hue, plane->hue);
+
+        plane->saturation = 0x1000;
+        drm_object_attach_property(&plane->base.base,
+                                   plane->props.saturation, plane->saturation);
+
+        plane->iturbt_709 = 0;
+        drm_object_attach_property(&plane->base.base,
+                                   plane->props.iturbt_709, plane->iturbt_709);
+
+        nv10_set_params(plane);
+        nv_wr32(dev, NV_PVIDEO_STOP, 1);
+        return;
+cleanup:
+        drm_plane_cleanup(&plane->base);
+err:
+        kfree(plane);
+        nv_error(dev, "Failed to create plane\n");
+}
+
+void
+nouveau_overlay_init(struct drm_device *device)
+{
+        struct nouveau_device *dev = nouveau_dev(device);
+        if (dev->chipset >= 0x10 && dev->chipset <= 0x40)
+                nv10_overlay_init(device);
+}
diff --git a/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c b/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c
index bf13db4e8631..cc4b208ce546 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/tvnv04.c
@@ -37,15 +37,18 @@
 
 #include <subdev/i2c.h>
 
-static struct i2c_board_info nv04_tv_encoder_info[] = {
+static struct nouveau_i2c_board_info nv04_tv_encoder_info[] = {
         {
-                I2C_BOARD_INFO("ch7006", 0x75),
-                .platform_data = &(struct ch7006_encoder_params) {
-                        CH7006_FORMAT_RGB24m12I, CH7006_CLOCK_MASTER,
-                        0, 0, 0,
-                        CH7006_SYNC_SLAVE, CH7006_SYNC_SEPARATED,
-                        CH7006_POUT_3_3V, CH7006_ACTIVE_HSYNC
-                }
+                {
+                        I2C_BOARD_INFO("ch7006", 0x75),
+                        .platform_data = &(struct ch7006_encoder_params) {
+                                CH7006_FORMAT_RGB24m12I, CH7006_CLOCK_MASTER,
+                                0, 0, 0,
+                                CH7006_SYNC_SLAVE, CH7006_SYNC_SEPARATED,
+                                CH7006_POUT_3_3V, CH7006_ACTIVE_HSYNC
+                        }
+                },
+                0
         },
         { }
 };
@@ -229,7 +232,8 @@ nv04_tv_create(struct drm_connector *connector, struct dcb_output *entry)
 
         /* Run the slave-specific initialization */
         ret = drm_i2c_encoder_init(dev, to_encoder_slave(encoder),
-                                   &port->adapter, &nv04_tv_encoder_info[type]);
+                                   &port->adapter,
+                                   &nv04_tv_encoder_info[type].dev);
         if (ret < 0)
                 goto fail_cleanup;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c
index 72055a35f845..3621e7f23477 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.c
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c
@@ -87,6 +87,7 @@ nouveau_abi16_swclass(struct nouveau_drm *drm)
         case NV_04:
                 return 0x006e;
         case NV_10:
+        case NV_11:
         case NV_20:
         case NV_30:
         case NV_40:
diff --git a/drivers/gpu/drm/nouveau/nouveau_acpi.c b/drivers/gpu/drm/nouveau/nouveau_acpi.c
index dd7d2e182719..f9a2df29a593 100644
--- a/drivers/gpu/drm/nouveau/nouveau_acpi.c
+++ b/drivers/gpu/drm/nouveau/nouveau_acpi.c
@@ -317,6 +317,16 @@ static bool nouveau_dsm_detect(void)
                         has_optimus = 1;
         }
 
+        while ((pdev = pci_get_class(PCI_CLASS_DISPLAY_3D << 8, pdev)) != NULL) {
+                vga_count++;
+
+                retval = nouveau_dsm_pci_probe(pdev);
+                if (retval & NOUVEAU_DSM_HAS_MUX)
+                        has_dsm |= 1;
+                if (retval & NOUVEAU_DSM_HAS_OPT)
+                        has_optimus = 1;
+        }
+
         /* find the optimus DSM or the old v1 DSM */
         if (has_optimus == 1) {
                 acpi_get_name(nouveau_dsm_priv.dhandle, ACPI_FULL_PATHNAME,
diff --git a/drivers/gpu/drm/nouveau/nouveau_agp.c b/drivers/gpu/drm/nouveau/nouveau_agp.c
index 6e7a55f93a85..2953c4e91e1a 100644
--- a/drivers/gpu/drm/nouveau/nouveau_agp.c
+++ b/drivers/gpu/drm/nouveau/nouveau_agp.c
@@ -11,10 +11,28 @@ MODULE_PARM_DESC(agpmode, "AGP mode (0 to disable AGP)");
 static int nouveau_agpmode = -1;
 module_param_named(agpmode, nouveau_agpmode, int, 0400);
 
+struct nouveau_agpmode_quirk {
+        u16 hostbridge_vendor;
+        u16 hostbridge_device;
+        u16 chip_vendor;
+        u16 chip_device;
+        int mode;
+};
+
+static struct nouveau_agpmode_quirk nouveau_agpmode_quirk_list[] = {
+        /* VIA Apollo PRO133x / GeForce FX 5600 Ultra, max agpmode 2, fdo #20341 */
+        { PCI_VENDOR_ID_VIA, 0x0691, PCI_VENDOR_ID_NVIDIA, 0x0311, 2 },
+
+        {},
+};
+
 static unsigned long
-get_agp_mode(struct nouveau_drm *drm, unsigned long mode)
+get_agp_mode(struct nouveau_drm *drm, const struct drm_agp_info *info)
 {
         struct nouveau_device *device = nv_device(drm->device);
+        struct nouveau_agpmode_quirk *quirk = nouveau_agpmode_quirk_list;
+        int agpmode = nouveau_agpmode;
+        unsigned long mode = info->mode;
 
         /*
          * FW seems to be broken on nv18, it makes the card lock up
@@ -24,11 +42,27 @@ get_agp_mode(struct nouveau_drm *drm, unsigned long mode)
                 mode &= ~PCI_AGP_COMMAND_FW;
 
         /*
+         * Go through the quirks list and adjust the agpmode accordingly.
+         */
+        while (agpmode == -1 && quirk->hostbridge_vendor) {
+                if (info->id_vendor == quirk->hostbridge_vendor &&
+                    info->id_device == quirk->hostbridge_device &&
+                    device->pdev->vendor == quirk->chip_vendor &&
+                    device->pdev->device == quirk->chip_device) {
+                        agpmode = quirk->mode;
+                        nv_info(device, "Forcing agp mode to %dX. Use agpmode to override.\n",
+                                agpmode);
+                        break;
+                }
+                ++quirk;
+        }
+
+        /*
          * AGP mode set in the command line.
          */
-        if (nouveau_agpmode > 0) {
+        if (agpmode > 0) {
                 bool agpv3 = mode & 0x8;
-                int rate = agpv3 ? nouveau_agpmode / 4 : nouveau_agpmode;
+                int rate = agpv3 ? agpmode / 4 : agpmode;
 
                 mode = (mode & ~0x7) | (rate & 0x7);
         }
@@ -90,7 +124,7 @@ nouveau_agp_reset(struct nouveau_drm *drm)
                 if (ret)
                         return;
 
-                mode.mode  = get_agp_mode(drm, info.mode);
+                mode.mode  = get_agp_mode(drm, &info);
                 mode.mode &= ~PCI_AGP_COMMAND_FW;
 
                 ret = drm_agp_enable(dev, mode);
@@ -139,7 +173,7 @@ nouveau_agp_init(struct nouveau_drm *drm)
         }
 
         /* see agp.h for the AGPSTAT_* modes available */
-        mode.mode = get_agp_mode(drm, info.mode);
+        mode.mode = get_agp_mode(drm, &info);
 
         ret = drm_agp_enable(dev, mode);
         if (ret) {
diff --git a/drivers/gpu/drm/nouveau/nouveau_backlight.c b/drivers/gpu/drm/nouveau/nouveau_backlight.c
index 2ffad2176b7f..630f6e84fc01 100644
--- a/drivers/gpu/drm/nouveau/nouveau_backlight.c
+++ b/drivers/gpu/drm/nouveau/nouveau_backlight.c
@@ -82,7 +82,7 @@ nv40_backlight_init(struct drm_connector *connector)
         memset(&props, 0, sizeof(struct backlight_properties));
         props.type = BACKLIGHT_RAW;
         props.max_brightness = 31;
-        bd = backlight_device_register("nv_backlight", &connector->kdev, drm,
+        bd = backlight_device_register("nv_backlight", connector->kdev, drm,
                                        &nv40_bl_ops, &props);
         if (IS_ERR(bd))
                 return PTR_ERR(bd);
@@ -204,7 +204,7 @@ nv50_backlight_init(struct drm_connector *connector)
         memset(&props, 0, sizeof(struct backlight_properties));
         props.type = BACKLIGHT_RAW;
         props.max_brightness = 100;
-        bd = backlight_device_register("nv_backlight", &connector->kdev,
+        bd = backlight_device_register("nv_backlight", connector->kdev,
                                        nv_encoder, ops, &props);
         if (IS_ERR(bd))
                 return PTR_ERR(bd);
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 4172854d4365..949ab0cbc4ab 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -269,7 +269,8 @@ set_placement_range(struct nouveau_bo *nvbo, uint32_t type)
         struct nouveau_fb *pfb = nouveau_fb(drm->device);
         u32 vram_pages = pfb->ram->size >> PAGE_SHIFT;
 
-        if (nv_device(drm->device)->card_type == NV_10 &&
+        if ((nv_device(drm->device)->card_type == NV_10 ||
+             nv_device(drm->device)->card_type == NV_11) &&
             nvbo->tile_mode && (type & TTM_PL_FLAG_VRAM) &&
             nvbo->bo.mem.num_pages < vram_pages / 4) {
                 /*
@@ -982,7 +983,7 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
                      bool no_wait_gpu, struct ttm_mem_reg *new_mem)
 {
         struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
-        struct nouveau_channel *chan = chan = drm->ttm.chan;
+        struct nouveau_channel *chan = drm->ttm.chan;
         struct nouveau_bo *nvbo = nouveau_bo(bo);
         struct ttm_mem_reg *old_mem = &bo->mem;
         int ret;
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index 2136d0038252..1674882d60d5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -100,6 +100,7 @@ static void
 nouveau_connector_destroy(struct drm_connector *connector)
 {
         struct nouveau_connector *nv_connector = nouveau_connector(connector);
+        nouveau_event_ref(NULL, &nv_connector->hpd_func);
         kfree(nv_connector->edid);
         drm_sysfs_connector_remove(connector);
         drm_connector_cleanup(connector);
@@ -214,9 +215,10 @@ nouveau_connector_set_encoder(struct drm_connector *connector,
         } else {
                 connector->doublescan_allowed = true;
                 if (nv_device(drm->device)->card_type == NV_20 ||
-                   (nv_device(drm->device)->card_type == NV_10 &&
-                    (dev->pdev->device & 0x0ff0) != 0x0100 &&
-                    (dev->pdev->device & 0x0ff0) != 0x0150))
+                    ((nv_device(drm->device)->card_type == NV_10 ||
+                      nv_device(drm->device)->card_type == NV_11) &&
+                     (dev->pdev->device & 0x0ff0) != 0x0100 &&
+                     (dev->pdev->device & 0x0ff0) != 0x0150))
                         /* HW is broken */
                         connector->interlace_allowed = false;
                 else
@@ -932,10 +934,9 @@ nouveau_connector_hotplug_work(struct work_struct *work)
 }
 
 static int
-nouveau_connector_hotplug(struct nouveau_eventh *event, int index)
+nouveau_connector_hotplug(void *data, int index)
 {
-        struct nouveau_connector *nv_connector =
-                container_of(event, struct nouveau_connector, hpd_func);
+        struct nouveau_connector *nv_connector = data;
         schedule_work(&nv_connector->hpd_work);
         return NVKM_EVENT_KEEP;
 }
@@ -1007,10 +1008,16 @@ nouveau_connector_create(struct drm_device *dev, int index)
 
                 ret = gpio->find(gpio, 0, hpd[ffs((entry & 0x07033000) >> 12)],
                                  DCB_GPIO_UNUSED, &nv_connector->hpd);
-                nv_connector->hpd_func.func = nouveau_connector_hotplug;
                 if (ret)
                         nv_connector->hpd.func = DCB_GPIO_UNUSED;
 
+                if (nv_connector->hpd.func != DCB_GPIO_UNUSED) {
+                        nouveau_event_new(gpio->events, nv_connector->hpd.line,
+                                          nouveau_connector_hotplug,
+                                          nv_connector,
+                                         &nv_connector->hpd_func);
+                }
+
                 nv_connector->type = nv_connector->dcb[0];
                 if (drm_conntype_from_dcb(nv_connector->type) ==
                                           DRM_MODE_CONNECTOR_Unknown) {
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.h b/drivers/gpu/drm/nouveau/nouveau_connector.h
index 6e399aad491a..264a778f473b 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.h
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.h
@@ -69,7 +69,7 @@ struct nouveau_connector {
 
         struct dcb_gpio_func hpd;
         struct work_struct hpd_work;
-        struct nouveau_eventh hpd_func;
+        struct nouveau_eventh *hpd_func;
 
         int dithering_mode;
         int dithering_depth;
@@ -107,7 +107,4 @@ nouveau_crtc_connector_get(struct nouveau_crtc *nv_crtc)
 struct drm_connector *
 nouveau_connector_create(struct drm_device *, int index);
 
-int
-nouveau_connector_bpp(struct drm_connector *);
-
 #endif /* __NOUVEAU_CONNECTOR_H__ */
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index bdd5cf71a24c..44642d9094e6 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -38,12 +38,85 @@
 
 #include "nouveau_fence.h"
 
-#include <subdev/bios/gpio.h>
-#include <subdev/gpio.h>
 #include <engine/disp.h>
 
 #include <core/class.h>
 
+static int
+nouveau_display_vblank_handler(void *data, int head)
+{
+        struct nouveau_drm *drm = data;
+        drm_handle_vblank(drm->dev, head);
+        return NVKM_EVENT_KEEP;
+}
+
+int
+nouveau_display_vblank_enable(struct drm_device *dev, int head)
+{
+        struct nouveau_display *disp = nouveau_display(dev);
+        if (disp) {
+                nouveau_event_get(disp->vblank[head]);
+                return 0;
+        }
+        return -EIO;
+}
+
+void
+nouveau_display_vblank_disable(struct drm_device *dev, int head)
+{
+        struct nouveau_display *disp = nouveau_display(dev);
+        if (disp)
+                nouveau_event_put(disp->vblank[head]);
+}
+
+static void
+nouveau_display_vblank_fini(struct drm_device *dev)
+{
+        struct nouveau_display *disp = nouveau_display(dev);
+        int i;
+
+        if (disp->vblank) {
+                for (i = 0; i < dev->mode_config.num_crtc; i++)
+                        nouveau_event_ref(NULL, &disp->vblank[i]);
+                kfree(disp->vblank);
+                disp->vblank = NULL;
+        }
+
+        drm_vblank_cleanup(dev);
+}
+
+static int
+nouveau_display_vblank_init(struct drm_device *dev)
+{
+        struct nouveau_display *disp = nouveau_display(dev);
+        struct nouveau_drm *drm = nouveau_drm(dev);
+        struct nouveau_disp *pdisp = nouveau_disp(drm->device);
+        int ret, i;
+
+        disp->vblank = kzalloc(dev->mode_config.num_crtc *
+                               sizeof(*disp->vblank), GFP_KERNEL);
+        if (!disp->vblank)
+                return -ENOMEM;
+
+        for (i = 0; i < dev->mode_config.num_crtc; i++) {
+                ret = nouveau_event_new(pdisp->vblank, i,
+                                        nouveau_display_vblank_handler,
+                                        drm, &disp->vblank[i]);
+                if (ret) {
+                        nouveau_display_vblank_fini(dev);
+                        return ret;
+                }
+        }
+
+        ret = drm_vblank_init(dev, dev->mode_config.num_crtc);
+        if (ret) {
+                nouveau_display_vblank_fini(dev);
+                return ret;
+        }
+
+        return 0;
+}
+
 static void
 nouveau_user_framebuffer_destroy(struct drm_framebuffer *drm_fb)
 {
@@ -227,9 +300,7 @@ static struct nouveau_drm_prop_enum_list dither_depth[] = {
 int
 nouveau_display_init(struct drm_device *dev)
 {
-        struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_display *disp = nouveau_display(dev);
-        struct nouveau_gpio *gpio = nouveau_gpio(drm->device);
         struct drm_connector *connector;
         int ret;
 
@@ -243,10 +314,7 @@ nouveau_display_init(struct drm_device *dev)
         /* enable hotplug interrupts */
         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
                 struct nouveau_connector *conn = nouveau_connector(connector);
-                if (gpio && conn->hpd.func != DCB_GPIO_UNUSED) {
-                        nouveau_event_get(gpio->events, conn->hpd.line,
-                                         &conn->hpd_func);
-                }
+                if (conn->hpd_func) nouveau_event_get(conn->hpd_func);
         }
 
         return ret;
@@ -255,18 +323,13 @@ nouveau_display_init(struct drm_device *dev)
 void
 nouveau_display_fini(struct drm_device *dev)
 {
-        struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_display *disp = nouveau_display(dev);
-        struct nouveau_gpio *gpio = nouveau_gpio(drm->device);
         struct drm_connector *connector;
 
         /* disable hotplug interrupts */
         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
                 struct nouveau_connector *conn = nouveau_connector(connector);
-                if (gpio && conn->hpd.func != DCB_GPIO_UNUSED) {
-                        nouveau_event_put(gpio->events, conn->hpd.line,
-                                         &conn->hpd_func);
-                }
+                if (conn->hpd_func) nouveau_event_put(conn->hpd_func);
         }
 
         drm_kms_helper_poll_disable(dev);
@@ -352,7 +415,7 @@ nouveau_display_create(struct drm_device *dev)
                 goto disp_create_err;
 
         if (dev->mode_config.num_crtc) {
-                ret = drm_vblank_init(dev, dev->mode_config.num_crtc);
+                ret = nouveau_display_vblank_init(dev);
                 if (ret)
                         goto vblank_err;
         }
@@ -374,7 +437,7 @@ nouveau_display_destroy(struct drm_device *dev)
         struct nouveau_display *disp = nouveau_display(dev);
 
         nouveau_backlight_exit(dev);
-        drm_vblank_cleanup(dev);
+        nouveau_display_vblank_fini(dev);
 
         drm_kms_helper_poll_fini(dev);
         drm_mode_config_cleanup(dev);
@@ -394,7 +457,7 @@ nouveau_display_suspend(struct drm_device *dev)
 
         nouveau_display_fini(dev);
 
-        NV_SUSPEND(drm, "unpinning framebuffer(s)...\n");
+        NV_INFO(drm, "unpinning framebuffer(s)...\n");
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                 struct nouveau_framebuffer *nouveau_fb;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.h b/drivers/gpu/drm/nouveau/nouveau_display.h
index 025c66f8e0ed..8bc8bab90e8d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.h
+++ b/drivers/gpu/drm/nouveau/nouveau_display.h
@@ -36,6 +36,8 @@ struct nouveau_display {
         int  (*init)(struct drm_device *);
         void (*fini)(struct drm_device *);
 
+        struct nouveau_eventh **vblank;
+
         struct drm_property *dithering_mode;
         struct drm_property *dithering_depth;
         struct drm_property *underscan_property;
@@ -59,6 +61,8 @@ void nouveau_display_fini(struct drm_device *dev);
 int  nouveau_display_suspend(struct drm_device *dev);
 void nouveau_display_repin(struct drm_device *dev);
 void nouveau_display_resume(struct drm_device *dev);
+int  nouveau_display_vblank_enable(struct drm_device *, int);
+void nouveau_display_vblank_disable(struct drm_device *, int);
 
 int  nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                             struct drm_pending_vblank_event *event,
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 428d818be775..2418b0de589e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -46,7 +46,8 @@
 #include "nouveau_gem.h"
 #include "nouveau_agp.h"
 #include "nouveau_vga.h"
-#include "nouveau_pm.h"
+#include "nouveau_sysfs.h"
+#include "nouveau_hwmon.h"
 #include "nouveau_acpi.h"
 #include "nouveau_bios.h"
 #include "nouveau_ioctl.h"
@@ -78,41 +79,6 @@ module_param_named(runpm, nouveau_runtime_pm, int, 0400);
 
 static struct drm_driver driver;
 
-static int
-nouveau_drm_vblank_handler(struct nouveau_eventh *event, int head)
-{
-        struct nouveau_drm *drm =
-                container_of(event, struct nouveau_drm, vblank[head]);
-        drm_handle_vblank(drm->dev, head);
-        return NVKM_EVENT_KEEP;
-}
-
-static int
-nouveau_drm_vblank_enable(struct drm_device *dev, int head)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_disp *pdisp = nouveau_disp(drm->device);
-
-        if (WARN_ON_ONCE(head > ARRAY_SIZE(drm->vblank)))
-                return -EIO;
-        WARN_ON_ONCE(drm->vblank[head].func);
-        drm->vblank[head].func = nouveau_drm_vblank_handler;
-        nouveau_event_get(pdisp->vblank, head, &drm->vblank[head]);
-        return 0;
-}
-
-static void
-nouveau_drm_vblank_disable(struct drm_device *dev, int head)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_disp *pdisp = nouveau_disp(drm->device);
-        if (drm->vblank[head].func)
-                nouveau_event_put(pdisp->vblank, head, &drm->vblank[head]);
-        else
-                WARN_ON_ONCE(1);
-        drm->vblank[head].func = NULL;
-}
-
 static u64
 nouveau_name(struct pci_dev *pdev)
 {
@@ -177,7 +143,8 @@ nouveau_accel_init(struct nouveau_drm *drm)
 
         /* initialise synchronisation routines */
         if      (device->card_type < NV_10) ret = nv04_fence_create(drm);
-        else if (device->chipset   <  0x17) ret = nv10_fence_create(drm);
+        else if (device->card_type < NV_11 ||
+                 device->chipset   <  0x17) ret = nv10_fence_create(drm);
         else if (device->card_type < NV_50) ret = nv17_fence_create(drm);
         else if (device->chipset   <  0x84) ret = nv50_fence_create(drm);
         else if (device->card_type < NV_C0) ret = nv84_fence_create(drm);
@@ -418,8 +385,8 @@ nouveau_drm_load(struct drm_device *dev, unsigned long flags)
                         goto fail_dispinit;
         }
 
-        nouveau_pm_init(dev);
-
+        nouveau_sysfs_init(dev);
+        nouveau_hwmon_init(dev);
         nouveau_accel_init(drm);
         nouveau_fbcon_init(dev);
 
@@ -455,8 +422,8 @@ nouveau_drm_unload(struct drm_device *dev)
         pm_runtime_get_sync(dev->dev);
         nouveau_fbcon_fini(dev);
         nouveau_accel_fini(drm);
-
-        nouveau_pm_fini(dev);
+        nouveau_hwmon_fini(dev);
+        nouveau_sysfs_fini(dev);
 
         if (dev->mode_config.num_crtc)
                 nouveau_display_fini(dev);
@@ -496,16 +463,16 @@ nouveau_do_suspend(struct drm_device *dev)
         int ret;
 
         if (dev->mode_config.num_crtc) {
-                NV_SUSPEND(drm, "suspending display...\n");
+                NV_INFO(drm, "suspending display...\n");
                 ret = nouveau_display_suspend(dev);
                 if (ret)
                         return ret;
         }
 
-        NV_SUSPEND(drm, "evicting buffers...\n");
+        NV_INFO(drm, "evicting buffers...\n");
         ttm_bo_evict_mm(&drm->ttm.bdev, TTM_PL_VRAM);
 
-        NV_SUSPEND(drm, "waiting for kernel channels to go idle...\n");
+        NV_INFO(drm, "waiting for kernel channels to go idle...\n");
         if (drm->cechan) {
                 ret = nouveau_channel_idle(drm->cechan);
                 if (ret)
@@ -518,7 +485,7 @@ nouveau_do_suspend(struct drm_device *dev)
                         return ret;
         }
 
-        NV_SUSPEND(drm, "suspending client object trees...\n");
+        NV_INFO(drm, "suspending client object trees...\n");
         if (drm->fence && nouveau_fence(drm)->suspend) {
                 if (!nouveau_fence(drm)->suspend(drm))
                         return -ENOMEM;
@@ -530,7 +497,7 @@ nouveau_do_suspend(struct drm_device *dev)
                         goto fail_client;
         }
 
-        NV_SUSPEND(drm, "suspending kernel object tree...\n");
+        NV_INFO(drm, "suspending kernel object tree...\n");
         ret = nouveau_client_fini(&drm->client.base, true);
         if (ret)
                 goto fail_client;
@@ -544,7 +511,7 @@ fail_client:
         }
 
         if (dev->mode_config.num_crtc) {
-                NV_SUSPEND(drm, "resuming display...\n");
+                NV_INFO(drm, "resuming display...\n");
                 nouveau_display_resume(dev);
         }
         return ret;
@@ -563,7 +530,6 @@ int nouveau_pmops_suspend(struct device *dev)
         if (drm_dev->mode_config.num_crtc)
                 nouveau_fbcon_set_suspend(drm_dev, 1);
 
-        nv_suspend_set_printk_level(NV_DBG_INFO);
         ret = nouveau_do_suspend(drm_dev);
         if (ret)
                 return ret;
@@ -571,8 +537,6 @@ int nouveau_pmops_suspend(struct device *dev)
         pci_save_state(pdev);
         pci_disable_device(pdev);
         pci_set_power_state(pdev, PCI_D3hot);
-        nv_suspend_set_printk_level(NV_DBG_DEBUG);
-
         return 0;
 }
 
@@ -582,15 +546,15 @@ nouveau_do_resume(struct drm_device *dev)
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_cli *cli;
 
-        NV_SUSPEND(drm, "re-enabling device...\n");
+        NV_INFO(drm, "re-enabling device...\n");
 
         nouveau_agp_reset(drm);
 
-        NV_SUSPEND(drm, "resuming kernel object tree...\n");
+        NV_INFO(drm, "resuming kernel object tree...\n");
         nouveau_client_init(&drm->client.base);
         nouveau_agp_init(drm);
 
-        NV_SUSPEND(drm, "resuming client object trees...\n");
+        NV_INFO(drm, "resuming client object trees...\n");
         if (drm->fence && nouveau_fence(drm)->resume)
                 nouveau_fence(drm)->resume(drm);
 
@@ -599,10 +563,9 @@ nouveau_do_resume(struct drm_device *dev)
         }
 
         nouveau_run_vbios_init(dev);
-        nouveau_pm_resume(dev);
 
         if (dev->mode_config.num_crtc) {
-                NV_SUSPEND(drm, "resuming display...\n");
+                NV_INFO(drm, "resuming display...\n");
                 nouveau_display_repin(dev);
         }
 
@@ -626,19 +589,15 @@ int nouveau_pmops_resume(struct device *dev)
                 return ret;
         pci_set_master(pdev);
 
-        nv_suspend_set_printk_level(NV_DBG_INFO);
         ret = nouveau_do_resume(drm_dev);
-        if (ret) {
-                nv_suspend_set_printk_level(NV_DBG_DEBUG);
+        if (ret)
                 return ret;
-        }
         if (drm_dev->mode_config.num_crtc)
                 nouveau_fbcon_set_suspend(drm_dev, 0);
 
         nouveau_fbcon_zfill_all(drm_dev);
         if (drm_dev->mode_config.num_crtc)
                 nouveau_display_resume(drm_dev);
-        nv_suspend_set_printk_level(NV_DBG_DEBUG);
         return 0;
 }
 
@@ -648,12 +607,10 @@ static int nouveau_pmops_freeze(struct device *dev)
         struct drm_device *drm_dev = pci_get_drvdata(pdev);
         int ret;
 
-        nv_suspend_set_printk_level(NV_DBG_INFO);
         if (drm_dev->mode_config.num_crtc)
                 nouveau_fbcon_set_suspend(drm_dev, 1);
 
         ret = nouveau_do_suspend(drm_dev);
-        nv_suspend_set_printk_level(NV_DBG_DEBUG);
         return ret;
 }
 
@@ -663,18 +620,14 @@ static int nouveau_pmops_thaw(struct device *dev)
         struct drm_device *drm_dev = pci_get_drvdata(pdev);
         int ret;
 
-        nv_suspend_set_printk_level(NV_DBG_INFO);
         ret = nouveau_do_resume(drm_dev);
-        if (ret) {
-                nv_suspend_set_printk_level(NV_DBG_DEBUG);
+        if (ret)
                 return ret;
-        }
         if (drm_dev->mode_config.num_crtc)
                 nouveau_fbcon_set_suspend(drm_dev, 0);
         nouveau_fbcon_zfill_all(drm_dev);
         if (drm_dev->mode_config.num_crtc)
                 nouveau_display_resume(drm_dev);
-        nv_suspend_set_printk_level(NV_DBG_DEBUG);
         return 0;
 }
 
@@ -816,8 +769,8 @@ driver = {
 #endif
 
         .get_vblank_counter = drm_vblank_count,
-        .enable_vblank = nouveau_drm_vblank_enable,
-        .disable_vblank = nouveau_drm_vblank_disable,
+        .enable_vblank = nouveau_display_vblank_enable,
+        .disable_vblank = nouveau_display_vblank_disable,
 
         .ioctls = nouveau_ioctls,
         .num_ioctls = ARRAY_SIZE(nouveau_ioctls),
@@ -878,6 +831,7 @@ static int nouveau_pmops_runtime_suspend(struct device *dev)
         if (nouveau_runtime_pm == 0)
                 return -EINVAL;
 
+        nv_debug_level(SILENT);
         drm_kms_helper_poll_disable(drm_dev);
         vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_OFF);
         nouveau_switcheroo_optimus_dsm();
@@ -914,6 +868,7 @@ static int nouveau_pmops_runtime_resume(struct device *dev)
         nv_mask(device, 0x88488, (1 << 25), (1 << 25));
         vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_ON);
         drm_dev->switch_power_state = DRM_SWITCH_POWER_ON;
+        nv_debug_level(NORMAL);
         return ret;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.h b/drivers/gpu/drm/nouveau/nouveau_drm.h
index 994fd6ec373b..71ed2dadae61 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.h
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.h
@@ -51,10 +51,11 @@ struct nouveau_drm_tile {
 };
 
 enum nouveau_drm_handle {
-        NVDRM_CLIENT = 0xffffffff,
-        NVDRM_DEVICE = 0xdddddddd,
-        NVDRM_PUSH   = 0xbbbb0000, /* |= client chid */
-        NVDRM_CHAN   = 0xcccc0000, /* |= client chid */
+        NVDRM_CLIENT  = 0xffffffff,
+        NVDRM_DEVICE  = 0xdddddddd,
+        NVDRM_CONTROL = 0xdddddddc,
+        NVDRM_PUSH    = 0xbbbb0000, /* |= client chid */
+        NVDRM_CHAN    = 0xcccc0000, /* |= client chid */
 };
 
 struct nouveau_cli {
@@ -127,10 +128,10 @@ struct nouveau_drm {
         struct nvbios vbios;
         struct nouveau_display *display;
         struct backlight_device *backlight;
-        struct nouveau_eventh vblank[4];
 
         /* power management */
-        struct nouveau_pm *pm;
+        struct nouveau_hwmon *hwmon;
+        struct nouveau_sysfs *sysfs;
 
         /* display power reference */
         bool have_disp_power_ref;
@@ -154,7 +155,6 @@ nouveau_dev(struct drm_device *dev)
 int nouveau_pmops_suspend(struct device *);
 int nouveau_pmops_resume(struct device *);
 
-#define NV_SUSPEND(cli, fmt, args...) nv_suspend((cli), fmt, ##args)
 #define NV_FATAL(cli, fmt, args...) nv_fatal((cli), fmt, ##args)
 #define NV_ERROR(cli, fmt, args...) nv_error((cli), fmt, ##args)
 #define NV_WARN(cli, fmt, args...) nv_warn((cli), fmt, ##args)
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index c80b519b513a..7903e0ed3c75 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -503,34 +503,45 @@ nouveau_fbcon_fini(struct drm_device *dev)
         drm->fbcon = NULL;
 }
 
-void nouveau_fbcon_save_disable_accel(struct drm_device *dev)
+void
+nouveau_fbcon_save_disable_accel(struct drm_device *dev)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-
-        drm->fbcon->saved_flags = drm->fbcon->helper.fbdev->flags;
-        drm->fbcon->helper.fbdev->flags |= FBINFO_HWACCEL_DISABLED;
+        if (drm->fbcon) {
+                drm->fbcon->saved_flags = drm->fbcon->helper.fbdev->flags;
+                drm->fbcon->helper.fbdev->flags |= FBINFO_HWACCEL_DISABLED;
+        }
 }
 
-void nouveau_fbcon_restore_accel(struct drm_device *dev)
+void
+nouveau_fbcon_restore_accel(struct drm_device *dev)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        drm->fbcon->helper.fbdev->flags = drm->fbcon->saved_flags;
+        if (drm->fbcon) {
+                drm->fbcon->helper.fbdev->flags = drm->fbcon->saved_flags;
+        }
 }
 
-void nouveau_fbcon_set_suspend(struct drm_device *dev, int state)
+void
+nouveau_fbcon_set_suspend(struct drm_device *dev, int state)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        console_lock();
-        if (state == 0)
-                nouveau_fbcon_save_disable_accel(dev);
-        fb_set_suspend(drm->fbcon->helper.fbdev, state);
-        if (state == 1)
-                nouveau_fbcon_restore_accel(dev);
-        console_unlock();
+        if (drm->fbcon) {
+                console_lock();
+                if (state == 0)
+                        nouveau_fbcon_save_disable_accel(dev);
+                fb_set_suspend(drm->fbcon->helper.fbdev, state);
+                if (state == 1)
+                        nouveau_fbcon_restore_accel(dev);
+                console_unlock();
+        }
 }
 
-void nouveau_fbcon_zfill_all(struct drm_device *dev)
+void
+nouveau_fbcon_zfill_all(struct drm_device *dev)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
-        nouveau_fbcon_zfill(dev, drm->fbcon);
+        if (drm->fbcon) {
+                nouveau_fbcon_zfill(dev, drm->fbcon);
+        }
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.c b/drivers/gpu/drm/nouveau/nouveau_fence.c
index be3149932c2d..34b82711e7c8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.c
@@ -165,17 +165,11 @@ nouveau_fence_done(struct nouveau_fence *fence)
         return !fence->channel;
 }
 
-struct nouveau_fence_uevent {
-        struct nouveau_eventh handler;
-        struct nouveau_fence_priv *priv;
-};
-
 static int
-nouveau_fence_wait_uevent_handler(struct nouveau_eventh *event, int index)
+nouveau_fence_wait_uevent_handler(void *data, int index)
 {
-        struct nouveau_fence_uevent *uevent =
-                container_of(event, struct nouveau_fence_uevent, handler);
-        wake_up_all(&uevent->priv->waiting);
+        struct nouveau_fence_priv *priv = data;
+        wake_up_all(&priv->waiting);
         return NVKM_EVENT_KEEP;
 }
 
@@ -186,13 +180,16 @@ nouveau_fence_wait_uevent(struct nouveau_fence *fence, bool intr)
         struct nouveau_channel *chan = fence->channel;
         struct nouveau_fifo *pfifo = nouveau_fifo(chan->drm->device);
         struct nouveau_fence_priv *priv = chan->drm->fence;
-        struct nouveau_fence_uevent uevent = {
-                .handler.func = nouveau_fence_wait_uevent_handler,
-                .priv = priv,
-        };
+        struct nouveau_eventh *handler;
         int ret = 0;
 
-        nouveau_event_get(pfifo->uevent, 0, &uevent.handler);
+        ret = nouveau_event_new(pfifo->uevent, 0,
+                                nouveau_fence_wait_uevent_handler,
+                                priv, &handler);
+        if (ret)
+                return ret;
+
+        nouveau_event_get(handler);
 
         if (fence->timeout) {
                 unsigned long timeout = fence->timeout - jiffies;
@@ -224,7 +221,7 @@ nouveau_fence_wait_uevent(struct nouveau_fence *fence, bool intr)
                 }
         }
 
-        nouveau_event_put(pfifo->uevent, 0, &uevent.handler);
+        nouveau_event_ref(NULL, &handler);
         if (unlikely(ret < 0))
                 return ret;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_pm.c b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
index 936b442a6ab7..38a4db5bfe21 100644
--- a/drivers/gpu/drm/nouveau/nouveau_pm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
@@ -32,369 +32,12 @@
 #include <drm/drmP.h>
 
 #include "nouveau_drm.h"
-#include "nouveau_pm.h"
+#include "nouveau_hwmon.h"
 
 #include <subdev/gpio.h>
 #include <subdev/timer.h>
 #include <subdev/therm.h>
 
-MODULE_PARM_DESC(perflvl, "Performance level (default: boot)");
-static char *nouveau_perflvl;
-module_param_named(perflvl, nouveau_perflvl, charp, 0400);
-
-MODULE_PARM_DESC(perflvl_wr, "Allow perflvl changes (warning: dangerous!)");
-static int nouveau_perflvl_wr;
-module_param_named(perflvl_wr, nouveau_perflvl_wr, int, 0400);
-
-static int
-nouveau_pm_perflvl_aux(struct drm_device *dev, struct nouveau_pm_level *perflvl,
-                       struct nouveau_pm_level *a, struct nouveau_pm_level *b)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
-        int ret;
-
-        /*XXX: not on all boards, we should control based on temperature
-         *     on recent boards..  or maybe on some other factor we don't
-         *     know about?
-         */
-        if (therm && therm->fan_set &&
-                a->fanspeed && b->fanspeed && b->fanspeed > a->fanspeed) {
-                ret = therm->fan_set(therm, perflvl->fanspeed);
-                if (ret && ret != -ENODEV) {
-                        NV_ERROR(drm, "fanspeed set failed: %d\n", ret);
-                }
-        }
-
-        if (pm->voltage.supported && pm->voltage_set) {
-                if (perflvl->volt_min && b->volt_min > a->volt_min) {
-                        ret = pm->voltage_set(dev, perflvl->volt_min);
-                        if (ret) {
-                                NV_ERROR(drm, "voltage set failed: %d\n", ret);
-                                return ret;
-                        }
-                }
-        }
-
-        return 0;
-}
-
-static int
-nouveau_pm_perflvl_set(struct drm_device *dev, struct nouveau_pm_level *perflvl)
-{
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        void *state;
-        int ret;
-
-        if (perflvl == pm->cur)
-                return 0;
-
-        ret = nouveau_pm_perflvl_aux(dev, perflvl, pm->cur, perflvl);
-        if (ret)
-                return ret;
-
-        state = pm->clocks_pre(dev, perflvl);
-        if (IS_ERR(state)) {
-                ret = PTR_ERR(state);
-                goto error;
-        }
-        ret = pm->clocks_set(dev, state);
-        if (ret)
-                goto error;
-
-        ret = nouveau_pm_perflvl_aux(dev, perflvl, perflvl, pm->cur);
-        if (ret)
-                return ret;
-
-        pm->cur = perflvl;
-        return 0;
-
-error:
-        /* restore the fan speed and voltage before leaving */
-        nouveau_pm_perflvl_aux(dev, perflvl, perflvl, pm->cur);
-        return ret;
-}
-
-void
-nouveau_pm_trigger(struct drm_device *dev)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_timer *ptimer = nouveau_timer(drm->device);
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        struct nouveau_pm_profile *profile = NULL;
-        struct nouveau_pm_level *perflvl = NULL;
-        int ret;
-
-        /* select power profile based on current power source */
-        if (power_supply_is_system_supplied())
-                profile = pm->profile_ac;
-        else
-                profile = pm->profile_dc;
-
-        if (profile != pm->profile) {
-                pm->profile->func->fini(pm->profile);
-                pm->profile = profile;
-                pm->profile->func->init(pm->profile);
-        }
-
-        /* select performance level based on profile */
-        perflvl = profile->func->select(profile);
-
-        /* change perflvl, if necessary */
-        if (perflvl != pm->cur) {
-                u64 time0 = ptimer->read(ptimer);
-
-                NV_INFO(drm, "setting performance level: %d", perflvl->id);
-                ret = nouveau_pm_perflvl_set(dev, perflvl);
-                if (ret)
-                        NV_INFO(drm, "> reclocking failed: %d\n\n", ret);
-
-                NV_INFO(drm, "> reclocking took %lluns\n\n",
-                             ptimer->read(ptimer) - time0);
-        }
-}
-
-static struct nouveau_pm_profile *
-profile_find(struct drm_device *dev, const char *string)
-{
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        struct nouveau_pm_profile *profile;
-
-        list_for_each_entry(profile, &pm->profiles, head) {
-                if (!strncmp(profile->name, string, sizeof(profile->name)))
-                        return profile;
-        }
-
-        return NULL;
-}
-
-static int
-nouveau_pm_profile_set(struct drm_device *dev, const char *profile)
-{
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        struct nouveau_pm_profile *ac = NULL, *dc = NULL;
-        char string[16], *cur = string, *ptr;
-
-        /* safety precaution, for now */
-        if (nouveau_perflvl_wr != 7777)
-                return -EPERM;
-
-        strncpy(string, profile, sizeof(string));
-        string[sizeof(string) - 1] = 0;
-        if ((ptr = strchr(string, '\n')))
-                *ptr = '\0';
-
-        ptr = strsep(&cur, ",");
-        if (ptr)
-                ac = profile_find(dev, ptr);
-
-        ptr = strsep(&cur, ",");
-        if (ptr)
-                dc = profile_find(dev, ptr);
-        else
-                dc = ac;
-
-        if (ac == NULL || dc == NULL)
-                return -EINVAL;
-
-        pm->profile_ac = ac;
-        pm->profile_dc = dc;
-        nouveau_pm_trigger(dev);
-        return 0;
-}
-
-static void
-nouveau_pm_static_dummy(struct nouveau_pm_profile *profile)
-{
-}
-
-static struct nouveau_pm_level *
-nouveau_pm_static_select(struct nouveau_pm_profile *profile)
-{
-        return container_of(profile, struct nouveau_pm_level, profile);
-}
-
-const struct nouveau_pm_profile_func nouveau_pm_static_profile_func = {
-        .destroy = nouveau_pm_static_dummy,
-        .init = nouveau_pm_static_dummy,
-        .fini = nouveau_pm_static_dummy,
-        .select = nouveau_pm_static_select,
-};
-
-static int
-nouveau_pm_perflvl_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        struct nouveau_therm *therm = nouveau_therm(drm->device);
-        int ret;
-
-        memset(perflvl, 0, sizeof(*perflvl));
-
-        if (pm->clocks_get) {
-                ret = pm->clocks_get(dev, perflvl);
-                if (ret)
-                        return ret;
-        }
-
-        if (pm->voltage.supported && pm->voltage_get) {
-                ret = pm->voltage_get(dev);
-                if (ret > 0) {
-                        perflvl->volt_min = ret;
-                        perflvl->volt_max = ret;
-                }
-        }
-
-        if (therm && therm->fan_get) {
-                ret = therm->fan_get(therm);
-                if (ret >= 0)
-                        perflvl->fanspeed = ret;
-        }
-
-        nouveau_mem_timing_read(dev, &perflvl->timing);
-        return 0;
-}
-
-static void
-nouveau_pm_perflvl_info(struct nouveau_pm_level *perflvl, char *ptr, int len)
-{
-        char c[16], s[16], v[32], f[16], m[16];
-
-        c[0] = '\0';
-        if (perflvl->core)
-                snprintf(c, sizeof(c), " core %dMHz", perflvl->core / 1000);
-
-        s[0] = '\0';
-        if (perflvl->shader)
-                snprintf(s, sizeof(s), " shader %dMHz", perflvl->shader / 1000);
-
-        m[0] = '\0';
-        if (perflvl->memory)
-                snprintf(m, sizeof(m), " memory %dMHz", perflvl->memory / 1000);
-
-        v[0] = '\0';
-        if (perflvl->volt_min && perflvl->volt_min != perflvl->volt_max) {
-                snprintf(v, sizeof(v), " voltage %dmV-%dmV",
-                         perflvl->volt_min / 1000, perflvl->volt_max / 1000);
-        } else
-        if (perflvl->volt_min) {
-                snprintf(v, sizeof(v), " voltage %dmV",
-                         perflvl->volt_min / 1000);
-        }
-
-        f[0] = '\0';
-        if (perflvl->fanspeed)
-                snprintf(f, sizeof(f), " fanspeed %d%%", perflvl->fanspeed);
-
-        snprintf(ptr, len, "%s%s%s%s%s\n", c, s, m, v, f);
-}
-
-static ssize_t
-nouveau_pm_get_perflvl_info(struct device *d,
-                            struct device_attribute *a, char *buf)
-{
-        struct nouveau_pm_level *perflvl =
-                container_of(a, struct nouveau_pm_level, dev_attr);
-        char *ptr = buf;
-        int len = PAGE_SIZE;
-
-        snprintf(ptr, len, "%d:", perflvl->id);
-        ptr += strlen(buf);
-        len -= strlen(buf);
-
-        nouveau_pm_perflvl_info(perflvl, ptr, len);
-        return strlen(buf);
-}
-
-static ssize_t
-nouveau_pm_get_perflvl(struct device *d, struct device_attribute *a, char *buf)
-{
-        struct drm_device *dev = pci_get_drvdata(to_pci_dev(d));
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        struct nouveau_pm_level cur;
-        int len = PAGE_SIZE, ret;
-        char *ptr = buf;
-
-        snprintf(ptr, len, "profile: %s, %s\nc:",
-                 pm->profile_ac->name, pm->profile_dc->name);
-        ptr += strlen(buf);
-        len -= strlen(buf);
-
-        ret = nouveau_pm_perflvl_get(dev, &cur);
-        if (ret == 0)
-                nouveau_pm_perflvl_info(&cur, ptr, len);
-        return strlen(buf);
-}
-
-static ssize_t
-nouveau_pm_set_perflvl(struct device *d, struct device_attribute *a,
-                       const char *buf, size_t count)
-{
-        struct drm_device *dev = pci_get_drvdata(to_pci_dev(d));
-        int ret;
-
-        ret = nouveau_pm_profile_set(dev, buf);
-        if (ret)
-                return ret;
-        return strlen(buf);
-}
-
-static DEVICE_ATTR(performance_level, S_IRUGO | S_IWUSR,
-                   nouveau_pm_get_perflvl, nouveau_pm_set_perflvl);
-
-static int
-nouveau_sysfs_init(struct drm_device *dev)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        struct device *d = &dev->pdev->dev;
-        int ret, i;
-
-        ret = device_create_file(d, &dev_attr_performance_level);
-        if (ret)
-                return ret;
-
-        for (i = 0; i < pm->nr_perflvl; i++) {
-                struct nouveau_pm_level *perflvl = &pm->perflvl[i];
-
-                perflvl->dev_attr.attr.name = perflvl->name;
-                perflvl->dev_attr.attr.mode = S_IRUGO;
-                perflvl->dev_attr.show = nouveau_pm_get_perflvl_info;
-                perflvl->dev_attr.store = NULL;
-                sysfs_attr_init(&perflvl->dev_attr.attr);
-
-                ret = device_create_file(d, &perflvl->dev_attr);
-                if (ret) {
-                        NV_ERROR(drm, "failed pervlvl %d sysfs: %d\n",
-                                 perflvl->id, i);
-                        perflvl->dev_attr.attr.name = NULL;
-                        nouveau_pm_fini(dev);
-                        return ret;
-                }
-        }
-
-        return 0;
-}
-
-static void
-nouveau_sysfs_fini(struct drm_device *dev)
-{
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        struct device *d = &dev->pdev->dev;
-        int i;
-
-        device_remove_file(d, &dev_attr_performance_level);
-        for (i = 0; i < pm->nr_perflvl; i++) {
-                struct nouveau_pm_level *pl = &pm->perflvl[i];
-
-                if (!pl->dev_attr.attr.name)
-                        break;
-
-                device_remove_file(d, &pl->dev_attr);
-        }
-}
-
 #if defined(CONFIG_HWMON) || (defined(MODULE) && defined(CONFIG_HWMON_MODULE))
 static ssize_t
 nouveau_hwmon_show_temp(struct device *d, struct device_attribute *a, char *buf)
@@ -778,9 +421,6 @@ nouveau_hwmon_set_pwm1(struct device *d, struct device_attribute *a,
         int ret = -ENODEV;
         long value;
 
-        if (nouveau_perflvl_wr != 7777)
-                return -EPERM;
-
         if (kstrtol(buf, 10, &value) == -EINVAL)
                 return -EINVAL;
 
@@ -919,17 +559,21 @@ static const struct attribute_group hwmon_pwm_fan_attrgroup = {
 };
 #endif
 
-static int
+int
 nouveau_hwmon_init(struct drm_device *dev)
 {
-        struct nouveau_pm *pm = nouveau_pm(dev);
-
 #if defined(CONFIG_HWMON) || (defined(MODULE) && defined(CONFIG_HWMON_MODULE))
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_therm *therm = nouveau_therm(drm->device);
+        struct nouveau_hwmon *hwmon;
         struct device *hwmon_dev;
         int ret = 0;
 
+        hwmon = drm->hwmon = kzalloc(sizeof(*hwmon), GFP_KERNEL);
+        if (!hwmon)
+                return -ENOMEM;
+        hwmon->dev = dev;
+
         if (!therm || !therm->temp_get || !therm->attr_get || !therm->attr_set)
                 return -ENODEV;
 
@@ -976,199 +620,37 @@ nouveau_hwmon_init(struct drm_device *dev)
                         goto error;
         }
 
-        pm->hwmon = hwmon_dev;
+        hwmon->hwmon = hwmon_dev;
 
         return 0;
 
 error:
         NV_ERROR(drm, "Unable to create some hwmon sysfs files: %d\n", ret);
         hwmon_device_unregister(hwmon_dev);
-        pm->hwmon = NULL;
+        hwmon->hwmon = NULL;
         return ret;
 #else
-        pm->hwmon = NULL;
+        hwmon->hwmon = NULL;
         return 0;
 #endif
 }
 
-static void
+void
 nouveau_hwmon_fini(struct drm_device *dev)
 {
 #if defined(CONFIG_HWMON) || (defined(MODULE) && defined(CONFIG_HWMON_MODULE))
-        struct nouveau_pm *pm = nouveau_pm(dev);
+        struct nouveau_hwmon *hwmon = nouveau_hwmon(dev);
 
-        if (pm->hwmon) {
-                sysfs_remove_group(&pm->hwmon->kobj, &hwmon_default_attrgroup);
-                sysfs_remove_group(&pm->hwmon->kobj, &hwmon_temp_attrgroup);
-                sysfs_remove_group(&pm->hwmon->kobj, &hwmon_pwm_fan_attrgroup);
-                sysfs_remove_group(&pm->hwmon->kobj, &hwmon_fan_rpm_attrgroup);
+        if (hwmon->hwmon) {
+                sysfs_remove_group(&hwmon->hwmon->kobj, &hwmon_default_attrgroup);
+                sysfs_remove_group(&hwmon->hwmon->kobj, &hwmon_temp_attrgroup);
+                sysfs_remove_group(&hwmon->hwmon->kobj, &hwmon_pwm_fan_attrgroup);
+                sysfs_remove_group(&hwmon->hwmon->kobj, &hwmon_fan_rpm_attrgroup);
 
-                hwmon_device_unregister(pm->hwmon);
+                hwmon_device_unregister(hwmon->hwmon);
         }
-#endif
-}
-
-#if defined(CONFIG_ACPI) && defined(CONFIG_POWER_SUPPLY)
-static int
-nouveau_pm_acpi_event(struct notifier_block *nb, unsigned long val, void *data)
-{
-        struct nouveau_pm *pm = container_of(nb, struct nouveau_pm, acpi_nb);
-        struct nouveau_drm *drm = nouveau_drm(pm->dev);
-        struct acpi_bus_event *entry = (struct acpi_bus_event *)data;
-
-        if (strcmp(entry->device_class, "ac_adapter") == 0) {
-                bool ac = power_supply_is_system_supplied();
 
-                NV_DEBUG(drm, "power supply changed: %s\n", ac ? "AC" : "DC");
-                nouveau_pm_trigger(pm->dev);
-        }
-
-        return NOTIFY_OK;
-}
+        nouveau_drm(dev)->hwmon = NULL;
+        kfree(hwmon);
 #endif
-
-int
-nouveau_pm_init(struct drm_device *dev)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_pm *pm;
-        char info[256];
-        int ret, i;
-
-        pm = drm->pm = kzalloc(sizeof(*pm), GFP_KERNEL);
-        if (!pm)
-                return -ENOMEM;
-
-        pm->dev = dev;
-
-        if (device->card_type < NV_40) {
-                pm->clocks_get = nv04_pm_clocks_get;
-                pm->clocks_pre = nv04_pm_clocks_pre;
-                pm->clocks_set = nv04_pm_clocks_set;
-                if (nouveau_gpio(drm->device)) {
-                        pm->voltage_get = nouveau_voltage_gpio_get;
-                        pm->voltage_set = nouveau_voltage_gpio_set;
-                }
-        } else
-        if (device->card_type < NV_50) {
-                pm->clocks_get = nv40_pm_clocks_get;
-                pm->clocks_pre = nv40_pm_clocks_pre;
-                pm->clocks_set = nv40_pm_clocks_set;
-                pm->voltage_get = nouveau_voltage_gpio_get;
-                pm->voltage_set = nouveau_voltage_gpio_set;
-        } else
-        if (device->card_type < NV_C0) {
-                if (device->chipset <  0xa3 ||
-                    device->chipset == 0xaa ||
-                    device->chipset == 0xac) {
-                        pm->clocks_get = nv50_pm_clocks_get;
-                        pm->clocks_pre = nv50_pm_clocks_pre;
-                        pm->clocks_set = nv50_pm_clocks_set;
-                } else {
-                        pm->clocks_get = nva3_pm_clocks_get;
-                        pm->clocks_pre = nva3_pm_clocks_pre;
-                        pm->clocks_set = nva3_pm_clocks_set;
-                }
-                pm->voltage_get = nouveau_voltage_gpio_get;
-                pm->voltage_set = nouveau_voltage_gpio_set;
-        } else
-        if (device->card_type < NV_E0) {
-                pm->clocks_get = nvc0_pm_clocks_get;
-                pm->clocks_pre = nvc0_pm_clocks_pre;
-                pm->clocks_set = nvc0_pm_clocks_set;
-                pm->voltage_get = nouveau_voltage_gpio_get;
-                pm->voltage_set = nouveau_voltage_gpio_set;
-        }
-
-
-        /* parse aux tables from vbios */
-        nouveau_volt_init(dev);
-
-        INIT_LIST_HEAD(&pm->profiles);
-
-        /* determine current ("boot") performance level */
-        ret = nouveau_pm_perflvl_get(dev, &pm->boot);
-        if (ret) {
-                NV_ERROR(drm, "failed to determine boot perflvl\n");
-                return ret;
-        }
-
-        strncpy(pm->boot.name, "boot", 4);
-        strncpy(pm->boot.profile.name, "boot", 4);
-        pm->boot.profile.func = &nouveau_pm_static_profile_func;
-
-        list_add(&pm->boot.profile.head, &pm->profiles);
-
-        pm->profile_ac = &pm->boot.profile;
-        pm->profile_dc = &pm->boot.profile;
-        pm->profile = &pm->boot.profile;
-        pm->cur = &pm->boot;
-
-        /* add performance levels from vbios */
-        nouveau_perf_init(dev);
-
-        /* display available performance levels */
-        NV_INFO(drm, "%d available performance level(s)\n", pm->nr_perflvl);
-        for (i = 0; i < pm->nr_perflvl; i++) {
-                nouveau_pm_perflvl_info(&pm->perflvl[i], info, sizeof(info));
-                NV_INFO(drm, "%d:%s", pm->perflvl[i].id, info);
-        }
-
-        nouveau_pm_perflvl_info(&pm->boot, info, sizeof(info));
-        NV_INFO(drm, "c:%s", info);
-
-        /* switch performance levels now if requested */
-        if (nouveau_perflvl != NULL)
-                nouveau_pm_profile_set(dev, nouveau_perflvl);
-
-        nouveau_sysfs_init(dev);
-        nouveau_hwmon_init(dev);
-#if defined(CONFIG_ACPI) && defined(CONFIG_POWER_SUPPLY)
-        pm->acpi_nb.notifier_call = nouveau_pm_acpi_event;
-        register_acpi_notifier(&pm->acpi_nb);
-#endif
-
-        return 0;
-}
-
-void
-nouveau_pm_fini(struct drm_device *dev)
-{
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        struct nouveau_pm_profile *profile, *tmp;
-
-        list_for_each_entry_safe(profile, tmp, &pm->profiles, head) {
-                list_del(&profile->head);
-                profile->func->destroy(profile);
-        }
-
-        if (pm->cur != &pm->boot)
-                nouveau_pm_perflvl_set(dev, &pm->boot);
-
-        nouveau_perf_fini(dev);
-        nouveau_volt_fini(dev);
-
-#if defined(CONFIG_ACPI) && defined(CONFIG_POWER_SUPPLY)
-        unregister_acpi_notifier(&pm->acpi_nb);
-#endif
-        nouveau_hwmon_fini(dev);
-        nouveau_sysfs_fini(dev);
-
-        nouveau_drm(dev)->pm = NULL;
-        kfree(pm);
-}
-
-void
-nouveau_pm_resume(struct drm_device *dev)
-{
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        struct nouveau_pm_level *perflvl;
-
-        if (!pm->cur || pm->cur == &pm->boot)
-                return;
-
-        perflvl = pm->cur;
-        pm->cur = &pm->boot;
-        nouveau_pm_perflvl_set(dev, perflvl);
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_hwmon.h b/drivers/gpu/drm/nouveau/nouveau_hwmon.h
new file mode 100644
index 000000000000..62ccbb39863c
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_hwmon.h
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2010 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#ifndef __NOUVEAU_PM_H__
+#define __NOUVEAU_PM_H__
+
+struct nouveau_hwmon {
+        struct drm_device *dev;
+        struct device *hwmon;
+};
+
+static inline struct nouveau_hwmon *
+nouveau_hwmon(struct drm_device *dev)
+{
+        return nouveau_drm(dev)->hwmon;
+}
+
+/* nouveau_hwmon.c */
+int  nouveau_hwmon_init(struct drm_device *dev);
+void nouveau_hwmon_fini(struct drm_device *dev);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_hwsq.h b/drivers/gpu/drm/nouveau/nouveau_hwsq.h
deleted file mode 100644
index 697687593a81..000000000000
--- a/drivers/gpu/drm/nouveau/nouveau_hwsq.h
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#ifndef __NOUVEAU_HWSQ_H__
-#define __NOUVEAU_HWSQ_H__
-
-struct hwsq_ucode {
-        u8 data[0x200];
-        union {
-                u8  *u08;
-                u16 *u16;
-                u32 *u32;
-        } ptr;
-        u16 len;
-
-        u32 reg;
-        u32 val;
-};
-
-static inline void
-hwsq_init(struct hwsq_ucode *hwsq)
-{
-        hwsq->ptr.u08 = hwsq->data;
-        hwsq->reg = 0xffffffff;
-        hwsq->val = 0xffffffff;
-}
-
-static inline void
-hwsq_fini(struct hwsq_ucode *hwsq)
-{
-        do {
-                *hwsq->ptr.u08++ = 0x7f;
-                hwsq->len = hwsq->ptr.u08 - hwsq->data;
-        } while (hwsq->len & 3);
-        hwsq->ptr.u08 = hwsq->data;
-}
-
-static inline void
-hwsq_usec(struct hwsq_ucode *hwsq, u8 usec)
-{
-        u32 shift = 0;
-        while (usec & ~3) {
-                usec >>= 2;
-                shift++;
-        }
-
-        *hwsq->ptr.u08++ = (shift << 2) | usec;
-}
-
-static inline void
-hwsq_setf(struct hwsq_ucode *hwsq, u8 flag, int val)
-{
-        flag += 0x80;
-        if (val >= 0)
-                flag += 0x20;
-        if (val >= 1)
-                flag += 0x20;
-        *hwsq->ptr.u08++ = flag;
-}
-
-static inline void
-hwsq_op5f(struct hwsq_ucode *hwsq, u8 v0, u8 v1)
-{
-        *hwsq->ptr.u08++ = 0x5f;
-        *hwsq->ptr.u08++ = v0;
-        *hwsq->ptr.u08++ = v1;
-}
-
-static inline void
-hwsq_wr32(struct hwsq_ucode *hwsq, u32 reg, u32 val)
-{
-        if (val != hwsq->val) {
-                if ((val & 0xffff0000) == (hwsq->val & 0xffff0000)) {
-                        *hwsq->ptr.u08++ = 0x42;
-                        *hwsq->ptr.u16++ = (val & 0x0000ffff);
-                } else {
-                        *hwsq->ptr.u08++ = 0xe2;
-                        *hwsq->ptr.u32++ = val;
-                }
-
-                hwsq->val = val;
-        }
-
-        if ((reg & 0xffff0000) == (hwsq->reg & 0xffff0000)) {
-                *hwsq->ptr.u08++ = 0x40;
-                *hwsq->ptr.u16++ = (reg & 0x0000ffff);
-        } else {
-                *hwsq->ptr.u08++ = 0xe0;
-                *hwsq->ptr.u32++ = reg;
-        }
-        hwsq->reg = reg;
-}
-
-#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_mem.c b/drivers/gpu/drm/nouveau/nouveau_mem.c
deleted file mode 100644
index 4f6a572f2258..000000000000
--- a/drivers/gpu/drm/nouveau/nouveau_mem.c
+++ /dev/null
@@ -1,647 +0,0 @@
-/*
- * Copyright (C) The Weather Channel, Inc.  2002.  All Rights Reserved.
- * Copyright 2005 Stephane Marchesin
- *
- * The Weather Channel (TM) funded Tungsten Graphics to develop the
- * initial release of the Radeon 8500 driver under the XFree86 license.
- * This notice must be preserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice (including the next
- * paragraph) shall be included in all copies or substantial portions of the
- * Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE AUTHORS AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
- * DEALINGS IN THE SOFTWARE.
- *
- * Authors:
- *    Ben Skeggs <bskeggs@redhat.com>
- *    Roy Spliet <r.spliet@student.tudelft.nl>
- */
-
-#include "nouveau_drm.h"
-#include "nouveau_pm.h"
-
-#include <subdev/fb.h>
-
-static int
-nv40_mem_timing_calc(struct drm_device *dev, u32 freq,
-                     struct nouveau_pm_tbl_entry *e, u8 len,
-                     struct nouveau_pm_memtiming *boot,
-                     struct nouveau_pm_memtiming *t)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-
-        t->reg[0] = (e->tRP << 24 | e->tRAS << 16 | e->tRFC << 8 | e->tRC);
-
-        /* XXX: I don't trust the -1's and +1's... they must come
-         *      from somewhere! */
-        t->reg[1] = (e->tWR + 2 + (t->tCWL - 1)) << 24 |
-                    1 << 16 |
-                    (e->tWTR + 2 + (t->tCWL - 1)) << 8 |
-                    (e->tCL + 2 - (t->tCWL - 1));
-
-        t->reg[2] = 0x20200000 |
-                    ((t->tCWL - 1) << 24 |
-                     e->tRRD << 16 |
-                     e->tRCDWR << 8 |
-                     e->tRCDRD);
-
-        NV_DEBUG(drm, "Entry %d: 220: %08x %08x %08x\n", t->id,
-                 t->reg[0], t->reg[1], t->reg[2]);
-        return 0;
-}
-
-static int
-nv50_mem_timing_calc(struct drm_device *dev, u32 freq,
-                     struct nouveau_pm_tbl_entry *e, u8 len,
-                     struct nouveau_pm_memtiming *boot,
-                     struct nouveau_pm_memtiming *t)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_fb *pfb = nouveau_fb(device);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct bit_entry P;
-        uint8_t unk18 = 1, unk20 = 0, unk21 = 0, tmp7_3;
-
-        if (bit_table(dev, 'P', &P))
-                return -EINVAL;
-
-        switch (min(len, (u8) 22)) {
-        case 22:
-                unk21 = e->tUNK_21;
-        case 21:
-                unk20 = e->tUNK_20;
-        case 20:
-                if (e->tCWL > 0)
-                        t->tCWL = e->tCWL;
-        case 19:
-                unk18 = e->tUNK_18;
-                break;
-        }
-
-        t->reg[0] = (e->tRP << 24 | e->tRAS << 16 | e->tRFC << 8 | e->tRC);
-
-        t->reg[1] = (e->tWR + 2 + (t->tCWL - 1)) << 24 |
-                                max(unk18, (u8) 1) << 16 |
-                                (e->tWTR + 2 + (t->tCWL - 1)) << 8;
-
-        t->reg[2] = ((t->tCWL - 1) << 24 |
-                    e->tRRD << 16 |
-                    e->tRCDWR << 8 |
-                    e->tRCDRD);
-
-        t->reg[4] = e->tUNK_13 << 8  | e->tUNK_13;
-
-        t->reg[5] = (e->tRFC << 24 | max(e->tRCDRD, e->tRCDWR) << 16 | e->tRP);
-
-        t->reg[8] = boot->reg[8] & 0xffffff00;
-
-        if (P.version == 1) {
-                t->reg[1] |= (e->tCL + 2 - (t->tCWL - 1));
-
-                t->reg[3] = (0x14 + e->tCL) << 24 |
-                            0x16 << 16 |
-                            (e->tCL - 1) << 8 |
-                            (e->tCL - 1);
-
-                t->reg[4] |= boot->reg[4] & 0xffff0000;
-
-                t->reg[6] = (0x33 - t->tCWL) << 16 |
-                            t->tCWL << 8 |
-                            (0x2e + e->tCL - t->tCWL);
-
-                t->reg[7] = 0x4000202 | (e->tCL - 1) << 16;
-
-                /* XXX: P.version == 1 only has DDR2 and GDDR3? */
-                if (pfb->ram->type == NV_MEM_TYPE_DDR2) {
-                        t->reg[5] |= (e->tCL + 3) << 8;
-                        t->reg[6] |= (t->tCWL - 2) << 8;
-                        t->reg[8] |= (e->tCL - 4);
-                } else {
-                        t->reg[5] |= (e->tCL + 2) << 8;
-                        t->reg[6] |= t->tCWL << 8;
-                        t->reg[8] |= (e->tCL - 2);
-                }
-        } else {
-                t->reg[1] |= (5 + e->tCL - (t->tCWL));
-
-                /* XXX: 0xb? 0x30? */
-                t->reg[3] = (0x30 + e->tCL) << 24 |
-                            (boot->reg[3] & 0x00ff0000)|
-                            (0xb + e->tCL) << 8 |
-                            (e->tCL - 1);
-
-                t->reg[4] |= (unk20 << 24 | unk21 << 16);
-
-                /* XXX: +6? */
-                t->reg[5] |= (t->tCWL + 6) << 8;
-
-                t->reg[6] = (0x5a + e->tCL) << 16 |
-                            (6 - e->tCL + t->tCWL) << 8 |
-                            (0x50 + e->tCL - t->tCWL);
-
-                tmp7_3 = (boot->reg[7] & 0xff000000) >> 24;
-                t->reg[7] = (tmp7_3 << 24) |
-                            ((tmp7_3 - 6 + e->tCL) << 16) |
-                            0x202;
-        }
-
-        NV_DEBUG(drm, "Entry %d: 220: %08x %08x %08x %08x\n", t->id,
-                 t->reg[0], t->reg[1], t->reg[2], t->reg[3]);
-        NV_DEBUG(drm, "         230: %08x %08x %08x %08x\n",
-                 t->reg[4], t->reg[5], t->reg[6], t->reg[7]);
-        NV_DEBUG(drm, "         240: %08x\n", t->reg[8]);
-        return 0;
-}
-
-static int
-nvc0_mem_timing_calc(struct drm_device *dev, u32 freq,
-                     struct nouveau_pm_tbl_entry *e, u8 len,
-                     struct nouveau_pm_memtiming *boot,
-                     struct nouveau_pm_memtiming *t)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-
-        if (e->tCWL > 0)
-                t->tCWL = e->tCWL;
-
-        t->reg[0] = (e->tRP << 24 | (e->tRAS & 0x7f) << 17 |
-                     e->tRFC << 8 | e->tRC);
-
-        t->reg[1] = (boot->reg[1] & 0xff000000) |
-                    (e->tRCDWR & 0x0f) << 20 |
-                    (e->tRCDRD & 0x0f) << 14 |
-                    (t->tCWL << 7) |
-                    (e->tCL & 0x0f);
-
-        t->reg[2] = (boot->reg[2] & 0xff0000ff) |
-                    e->tWR << 16 | e->tWTR << 8;
-
-        t->reg[3] = (e->tUNK_20 & 0x1f) << 9 |
-                    (e->tUNK_21 & 0xf) << 5 |
-                    (e->tUNK_13 & 0x1f);
-
-        t->reg[4] = (boot->reg[4] & 0xfff00fff) |
-                    (e->tRRD&0x1f) << 15;
-
-        NV_DEBUG(drm, "Entry %d: 290: %08x %08x %08x %08x\n", t->id,
-                 t->reg[0], t->reg[1], t->reg[2], t->reg[3]);
-        NV_DEBUG(drm, "         2a0: %08x\n", t->reg[4]);
-        return 0;
-}
-
-/**
- * MR generation methods
- */
-
-static int
-nouveau_mem_ddr2_mr(struct drm_device *dev, u32 freq,
-                    struct nouveau_pm_tbl_entry *e, u8 len,
-                    struct nouveau_pm_memtiming *boot,
-                    struct nouveau_pm_memtiming *t)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-
-        t->drive_strength = 0;
-        if (len < 15) {
-                t->odt = boot->odt;
-        } else {
-                t->odt = e->RAM_FT1 & 0x07;
-        }
-
-        if (e->tCL >= NV_MEM_CL_DDR2_MAX) {
-                NV_WARN(drm, "(%u) Invalid tCL: %u", t->id, e->tCL);
-                return -ERANGE;
-        }
-
-        if (e->tWR >= NV_MEM_WR_DDR2_MAX) {
-                NV_WARN(drm, "(%u) Invalid tWR: %u", t->id, e->tWR);
-                return -ERANGE;
-        }
-
-        if (t->odt > 3) {
-                NV_WARN(drm, "(%u) Invalid odt value, assuming disabled: %x",
-                        t->id, t->odt);
-                t->odt = 0;
-        }
-
-        t->mr[0] = (boot->mr[0] & 0x100f) |
-                   (e->tCL) << 4 |
-                   (e->tWR - 1) << 9;
-        t->mr[1] = (boot->mr[1] & 0x101fbb) |
-                   (t->odt & 0x1) << 2 |
-                   (t->odt & 0x2) << 5;
-
-        NV_DEBUG(drm, "(%u) MR: %08x", t->id, t->mr[0]);
-        return 0;
-}
-
-static const uint8_t nv_mem_wr_lut_ddr3[NV_MEM_WR_DDR3_MAX] = {
-        0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 5, 6, 6, 7, 7, 0, 0};
-
-static int
-nouveau_mem_ddr3_mr(struct drm_device *dev, u32 freq,
-                    struct nouveau_pm_tbl_entry *e, u8 len,
-                    struct nouveau_pm_memtiming *boot,
-                    struct nouveau_pm_memtiming *t)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        u8 cl = e->tCL - 4;
-
-        t->drive_strength = 0;
-        if (len < 15) {
-                t->odt = boot->odt;
-        } else {
-                t->odt = e->RAM_FT1 & 0x07;
-        }
-
-        if (e->tCL >= NV_MEM_CL_DDR3_MAX || e->tCL < 4) {
-                NV_WARN(drm, "(%u) Invalid tCL: %u", t->id, e->tCL);
-                return -ERANGE;
-        }
-
-        if (e->tWR >= NV_MEM_WR_DDR3_MAX || e->tWR < 4) {
-                NV_WARN(drm, "(%u) Invalid tWR: %u", t->id, e->tWR);
-                return -ERANGE;
-        }
-
-        if (e->tCWL < 5) {
-                NV_WARN(drm, "(%u) Invalid tCWL: %u", t->id, e->tCWL);
-                return -ERANGE;
-        }
-
-        t->mr[0] = (boot->mr[0] & 0x180b) |
-                   /* CAS */
-                   (cl & 0x7) << 4 |
-                   (cl & 0x8) >> 1 |
-                   (nv_mem_wr_lut_ddr3[e->tWR]) << 9;
-        t->mr[1] = (boot->mr[1] & 0x101dbb) |
-                   (t->odt & 0x1) << 2 |
-                   (t->odt & 0x2) << 5 |
-                   (t->odt & 0x4) << 7;
-        t->mr[2] = (boot->mr[2] & 0x20ffb7) | (e->tCWL - 5) << 3;
-
-        NV_DEBUG(drm, "(%u) MR: %08x %08x", t->id, t->mr[0], t->mr[2]);
-        return 0;
-}
-
-static const uint8_t nv_mem_cl_lut_gddr3[NV_MEM_CL_GDDR3_MAX] = {
-        0, 0, 0, 0, 4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11};
-static const uint8_t nv_mem_wr_lut_gddr3[NV_MEM_WR_GDDR3_MAX] = {
-        0, 0, 0, 0, 0, 2, 3, 8, 9, 10, 11, 0, 0, 1, 1, 0, 3};
-
-static int
-nouveau_mem_gddr3_mr(struct drm_device *dev, u32 freq,
-                     struct nouveau_pm_tbl_entry *e, u8 len,
-                     struct nouveau_pm_memtiming *boot,
-                     struct nouveau_pm_memtiming *t)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-
-        if (len < 15) {
-                t->drive_strength = boot->drive_strength;
-                t->odt = boot->odt;
-        } else {
-                t->drive_strength = (e->RAM_FT1 & 0x30) >> 4;
-                t->odt = e->RAM_FT1 & 0x07;
-        }
-
-        if (e->tCL >= NV_MEM_CL_GDDR3_MAX) {
-                NV_WARN(drm, "(%u) Invalid tCL: %u", t->id, e->tCL);
-                return -ERANGE;
-        }
-
-        if (e->tWR >= NV_MEM_WR_GDDR3_MAX) {
-                NV_WARN(drm, "(%u) Invalid tWR: %u", t->id, e->tWR);
-                return -ERANGE;
-        }
-
-        if (t->odt > 3) {
-                NV_WARN(drm, "(%u) Invalid odt value, assuming autocal: %x",
-                        t->id, t->odt);
-                t->odt = 0;
-        }
-
-        t->mr[0] = (boot->mr[0] & 0xe0b) |
-                   /* CAS */
-                   ((nv_mem_cl_lut_gddr3[e->tCL] & 0x7) << 4) |
-                   ((nv_mem_cl_lut_gddr3[e->tCL] & 0x8) >> 2);
-        t->mr[1] = (boot->mr[1] & 0x100f40) | t->drive_strength |
-                   (t->odt << 2) |
-                   (nv_mem_wr_lut_gddr3[e->tWR] & 0xf) << 4;
-        t->mr[2] = boot->mr[2];
-
-        NV_DEBUG(drm, "(%u) MR: %08x %08x %08x", t->id,
-                      t->mr[0], t->mr[1], t->mr[2]);
-        return 0;
-}
-
-static int
-nouveau_mem_gddr5_mr(struct drm_device *dev, u32 freq,
-                     struct nouveau_pm_tbl_entry *e, u8 len,
-                     struct nouveau_pm_memtiming *boot,
-                     struct nouveau_pm_memtiming *t)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-
-        if (len < 15) {
-                t->drive_strength = boot->drive_strength;
-                t->odt = boot->odt;
-        } else {
-                t->drive_strength = (e->RAM_FT1 & 0x30) >> 4;
-                t->odt = e->RAM_FT1 & 0x03;
-        }
-
-        if (e->tCL >= NV_MEM_CL_GDDR5_MAX) {
-                NV_WARN(drm, "(%u) Invalid tCL: %u", t->id, e->tCL);
-                return -ERANGE;
-        }
-
-        if (e->tWR >= NV_MEM_WR_GDDR5_MAX) {
-                NV_WARN(drm, "(%u) Invalid tWR: %u", t->id, e->tWR);
-                return -ERANGE;
-        }
-
-        if (t->odt > 3) {
-                NV_WARN(drm, "(%u) Invalid odt value, assuming autocal: %x",
-                        t->id, t->odt);
-                t->odt = 0;
-        }
-
-        t->mr[0] = (boot->mr[0] & 0x007) |
-                   ((e->tCL - 5) << 3) |
-                   ((e->tWR - 4) << 8);
-        t->mr[1] = (boot->mr[1] & 0x1007f0) |
-                   t->drive_strength |
-                   (t->odt << 2);
-
-        NV_DEBUG(drm, "(%u) MR: %08x %08x", t->id, t->mr[0], t->mr[1]);
-        return 0;
-}
-
-int
-nouveau_mem_timing_calc(struct drm_device *dev, u32 freq,
-                        struct nouveau_pm_memtiming *t)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_fb *pfb = nouveau_fb(device);
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        struct nouveau_pm_memtiming *boot = &pm->boot.timing;
-        struct nouveau_pm_tbl_entry *e;
-        u8 ver, len, *ptr, *ramcfg;
-        int ret;
-
-        ptr = nouveau_perf_timing(dev, freq, &ver, &len);
-        if (!ptr || ptr[0] == 0x00) {
-                *t = *boot;
-                return 0;
-        }
-        e = (struct nouveau_pm_tbl_entry *)ptr;
-
-        t->tCWL = boot->tCWL;
-
-        switch (device->card_type) {
-        case NV_40:
-                ret = nv40_mem_timing_calc(dev, freq, e, len, boot, t);
-                break;
-        case NV_50:
-                ret = nv50_mem_timing_calc(dev, freq, e, len, boot, t);
-                break;
-        case NV_C0:
-        case NV_D0:
-                ret = nvc0_mem_timing_calc(dev, freq, e, len, boot, t);
-                break;
-        default:
-                ret = -ENODEV;
-                break;
-        }
-
-        switch (pfb->ram->type * !ret) {
-        case NV_MEM_TYPE_GDDR3:
-                ret = nouveau_mem_gddr3_mr(dev, freq, e, len, boot, t);
-                break;
-        case NV_MEM_TYPE_GDDR5:
-                ret = nouveau_mem_gddr5_mr(dev, freq, e, len, boot, t);
-                break;
-        case NV_MEM_TYPE_DDR2:
-                ret = nouveau_mem_ddr2_mr(dev, freq, e, len, boot, t);
-                break;
-        case NV_MEM_TYPE_DDR3:
-                ret = nouveau_mem_ddr3_mr(dev, freq, e, len, boot, t);
-                break;
-        default:
-                ret = -EINVAL;
-                break;
-        }
-
-        ramcfg = nouveau_perf_ramcfg(dev, freq, &ver, &len);
-        if (ramcfg) {
-                int dll_off;
-
-                if (ver == 0x00)
-                        dll_off = !!(ramcfg[3] & 0x04);
-                else
-                        dll_off = !!(ramcfg[2] & 0x40);
-
-                switch (pfb->ram->type) {
-                case NV_MEM_TYPE_GDDR3:
-                        t->mr[1] &= ~0x00000040;
-                        t->mr[1] |=  0x00000040 * dll_off;
-                        break;
-                default:
-                        t->mr[1] &= ~0x00000001;
-                        t->mr[1] |=  0x00000001 * dll_off;
-                        break;
-                }
-        }
-
-        return ret;
-}
-
-void
-nouveau_mem_timing_read(struct drm_device *dev, struct nouveau_pm_memtiming *t)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_fb *pfb = nouveau_fb(device);
-        u32 timing_base, timing_regs, mr_base;
-        int i;
-
-        if (device->card_type >= 0xC0) {
-                timing_base = 0x10f290;
-                mr_base = 0x10f300;
-        } else {
-                timing_base = 0x100220;
-                mr_base = 0x1002c0;
-        }
-
-        t->id = -1;
-
-        switch (device->card_type) {
-        case NV_50:
-                timing_regs = 9;
-                break;
-        case NV_C0:
-        case NV_D0:
-                timing_regs = 5;
-                break;
-        case NV_30:
-        case NV_40:
-                timing_regs = 3;
-                break;
-        default:
-                timing_regs = 0;
-                return;
-        }
-        for(i = 0; i < timing_regs; i++)
-                t->reg[i] = nv_rd32(device, timing_base + (0x04 * i));
-
-        t->tCWL = 0;
-        if (device->card_type < NV_C0) {
-                t->tCWL = ((nv_rd32(device, 0x100228) & 0x0f000000) >> 24) + 1;
-        } else if (device->card_type <= NV_D0) {
-                t->tCWL = ((nv_rd32(device, 0x10f294) & 0x00000f80) >> 7);
-        }
-
-        t->mr[0] = nv_rd32(device, mr_base);
-        t->mr[1] = nv_rd32(device, mr_base + 0x04);
-        t->mr[2] = nv_rd32(device, mr_base + 0x20);
-        t->mr[3] = nv_rd32(device, mr_base + 0x24);
-
-        t->odt = 0;
-        t->drive_strength = 0;
-
-        switch (pfb->ram->type) {
-        case NV_MEM_TYPE_DDR3:
-                t->odt |= (t->mr[1] & 0x200) >> 7;
-        case NV_MEM_TYPE_DDR2:
-                t->odt |= (t->mr[1] & 0x04) >> 2 |
-                          (t->mr[1] & 0x40) >> 5;
-                break;
-        case NV_MEM_TYPE_GDDR3:
-        case NV_MEM_TYPE_GDDR5:
-                t->drive_strength = t->mr[1] & 0x03;
-                t->odt = (t->mr[1] & 0x0c) >> 2;
-                break;
-        default:
-                break;
-        }
-}
-
-int
-nouveau_mem_exec(struct nouveau_mem_exec_func *exec,
-                 struct nouveau_pm_level *perflvl)
-{
-        struct nouveau_drm *drm = nouveau_drm(exec->dev);
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        struct nouveau_fb *pfb = nouveau_fb(device);
-        struct nouveau_pm_memtiming *info = &perflvl->timing;
-        u32 tMRD = 1000, tCKSRE = 0, tCKSRX = 0, tXS = 0, tDLLK = 0;
-        u32 mr[3] = { info->mr[0], info->mr[1], info->mr[2] };
-        u32 mr1_dlloff;
-
-        switch (pfb->ram->type) {
-        case NV_MEM_TYPE_DDR2:
-                tDLLK = 2000;
-                mr1_dlloff = 0x00000001;
-                break;
-        case NV_MEM_TYPE_DDR3:
-                tDLLK = 12000;
-                tCKSRE = 2000;
-                tXS = 1000;
-                mr1_dlloff = 0x00000001;
-                break;
-        case NV_MEM_TYPE_GDDR3:
-                tDLLK = 40000;
-                mr1_dlloff = 0x00000040;
-                break;
-        default:
-                NV_ERROR(drm, "cannot reclock unsupported memtype\n");
-                return -ENODEV;
-        }
-
-        /* fetch current MRs */
-        switch (pfb->ram->type) {
-        case NV_MEM_TYPE_GDDR3:
-        case NV_MEM_TYPE_DDR3:
-                mr[2] = exec->mrg(exec, 2);
-        default:
-                mr[1] = exec->mrg(exec, 1);
-                mr[0] = exec->mrg(exec, 0);
-                break;
-        }
-
-        /* DLL 'on' -> DLL 'off' mode, disable before entering self-refresh  */
-        if (!(mr[1] & mr1_dlloff) && (info->mr[1] & mr1_dlloff)) {
-                exec->precharge(exec);
-                exec->mrs (exec, 1, mr[1] | mr1_dlloff);
-                exec->wait(exec, tMRD);
-        }
-
-        /* enter self-refresh mode */
-        exec->precharge(exec);
-        exec->refresh(exec);
-        exec->refresh(exec);
-        exec->refresh_auto(exec, false);
-        exec->refresh_self(exec, true);
-        exec->wait(exec, tCKSRE);
-
-        /* modify input clock frequency */
-        exec->clock_set(exec);
-
-        /* exit self-refresh mode */
-        exec->wait(exec, tCKSRX);
-        exec->precharge(exec);
-        exec->refresh_self(exec, false);
-        exec->refresh_auto(exec, true);
-        exec->wait(exec, tXS);
-        exec->wait(exec, tXS);
-
-        /* update MRs */
-        if (mr[2] != info->mr[2]) {
-                exec->mrs (exec, 2, info->mr[2]);
-                exec->wait(exec, tMRD);
-        }
-
-        if (mr[1] != info->mr[1]) {
-                /* need to keep DLL off until later, at least on GDDR3 */
-                exec->mrs (exec, 1, info->mr[1] | (mr[1] & mr1_dlloff));
-                exec->wait(exec, tMRD);
-        }
-
-        if (mr[0] != info->mr[0]) {
-                exec->mrs (exec, 0, info->mr[0]);
-                exec->wait(exec, tMRD);
-        }
-
-        /* update PFB timing registers */
-        exec->timing_set(exec);
-
-        /* DLL (enable + ) reset */
-        if (!(info->mr[1] & mr1_dlloff)) {
-                if (mr[1] & mr1_dlloff) {
-                        exec->mrs (exec, 1, info->mr[1]);
-                        exec->wait(exec, tMRD);
-                }
-                exec->mrs (exec, 0, info->mr[0] | 0x00000100);
-                exec->wait(exec, tMRD);
-                exec->mrs (exec, 0, info->mr[0] | 0x00000000);
-                exec->wait(exec, tMRD);
-                exec->wait(exec, tDLLK);
-                if (pfb->ram->type == NV_MEM_TYPE_GDDR3)
-                        exec->precharge(exec);
-        }
-
-        return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_perf.c b/drivers/gpu/drm/nouveau/nouveau_perf.c
deleted file mode 100644
index 4fe883c54918..000000000000
--- a/drivers/gpu/drm/nouveau/nouveau_perf.c
+++ /dev/null
@@ -1,416 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include <drm/drmP.h>
-
-#include "nouveau_drm.h"
-#include "nouveau_reg.h"
-#include "nouveau_pm.h"
-
-static u8 *
-nouveau_perf_table(struct drm_device *dev, u8 *ver)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nvbios *bios = &drm->vbios;
-        struct bit_entry P;
-
-        if (!bit_table(dev, 'P', &P) && P.version && P.version <= 2) {
-                u8 *perf = ROMPTR(dev, P.data[0]);
-                if (perf) {
-                        *ver = perf[0];
-                        return perf;
-                }
-        }
-
-        if (bios->type == NVBIOS_BMP) {
-                if (bios->data[bios->offset + 6] >= 0x25) {
-                        u8 *perf = ROMPTR(dev, bios->data[bios->offset + 0x94]);
-                        if (perf) {
-                                *ver = perf[1];
-                                return perf;
-                        }
-                }
-        }
-
-        return NULL;
-}
-
-static u8 *
-nouveau_perf_entry(struct drm_device *dev, int idx,
-                   u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
-{
-        u8 *perf = nouveau_perf_table(dev, ver);
-        if (perf) {
-                if (*ver >= 0x12 && *ver < 0x20 && idx < perf[2]) {
-                        *hdr = perf[3];
-                        *cnt = 0;
-                        *len = 0;
-                        return perf + perf[0] + idx * perf[3];
-                } else
-                if (*ver >= 0x20 && *ver < 0x40 && idx < perf[2]) {
-                        *hdr = perf[3];
-                        *cnt = perf[4];
-                        *len = perf[5];
-                        return perf + perf[1] + idx * (*hdr + (*cnt * *len));
-                } else
-                if (*ver >= 0x40 && *ver < 0x41 && idx < perf[5]) {
-                        *hdr = perf[2];
-                        *cnt = perf[4];
-                        *len = perf[3];
-                        return perf + perf[1] + idx * (*hdr + (*cnt * *len));
-                }
-        }
-        return NULL;
-}
-
-u8 *
-nouveau_perf_rammap(struct drm_device *dev, u32 freq,
-                    u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct bit_entry P;
-        u8 *perf, i = 0;
-
-        if (!bit_table(dev, 'P', &P) && P.version == 2) {
-                u8 *rammap = ROMPTR(dev, P.data[4]);
-                if (rammap) {
-                        u8 *ramcfg = rammap + rammap[1];
-
-                        *ver = rammap[0];
-                        *hdr = rammap[2];
-                        *cnt = rammap[4];
-                        *len = rammap[3];
-
-                        freq /= 1000;
-                        for (i = 0; i < rammap[5]; i++) {
-                                if (freq >= ROM16(ramcfg[0]) &&
-                                    freq <= ROM16(ramcfg[2]))
-                                        return ramcfg;
-
-                                ramcfg += *hdr + (*cnt * *len);
-                        }
-                }
-
-                return NULL;
-        }
-
-        if (nv_device(drm->device)->chipset == 0x49 ||
-            nv_device(drm->device)->chipset == 0x4b)
-                freq /= 2;
-
-        while ((perf = nouveau_perf_entry(dev, i++, ver, hdr, cnt, len))) {
-                if (*ver >= 0x20 && *ver < 0x25) {
-                        if (perf[0] != 0xff && freq <= ROM16(perf[11]) * 1000)
-                                break;
-                } else
-                if (*ver >= 0x25 && *ver < 0x40) {
-                        if (perf[0] != 0xff && freq <= ROM16(perf[12]) * 1000)
-                                break;
-                }
-        }
-
-        if (perf) {
-                u8 *ramcfg = perf + *hdr;
-                *ver = 0x00;
-                *hdr = 0;
-                return ramcfg;
-        }
-
-        return NULL;
-}
-
-u8 *
-nouveau_perf_ramcfg(struct drm_device *dev, u32 freq, u8 *ver, u8 *len)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nvbios *bios = &drm->vbios;
-        u8 strap, hdr, cnt;
-        u8 *rammap;
-
-        strap = (nv_rd32(device, 0x101000) & 0x0000003c) >> 2;
-        if (bios->ram_restrict_tbl_ptr)
-                strap = bios->data[bios->ram_restrict_tbl_ptr + strap];
-
-        rammap = nouveau_perf_rammap(dev, freq, ver, &hdr, &cnt, len);
-        if (rammap && strap < cnt)
-                return rammap + hdr + (strap * *len);
-
-        return NULL;
-}
-
-u8 *
-nouveau_perf_timing(struct drm_device *dev, u32 freq, u8 *ver, u8 *len)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nvbios *bios = &drm->vbios;
-        struct bit_entry P;
-        u8 *perf, *timing = NULL;
-        u8 i = 0, hdr, cnt;
-
-        if (bios->type == NVBIOS_BMP) {
-                while ((perf = nouveau_perf_entry(dev, i++, ver, &hdr, &cnt,
-                                                  len)) && *ver == 0x15) {
-                        if (freq <= ROM32(perf[5]) * 20) {
-                                *ver = 0x00;
-                                *len = 14;
-                                return perf + 41;
-                        }
-                }
-                return NULL;
-        }
-
-        if (!bit_table(dev, 'P', &P)) {
-                if (P.version == 1)
-                        timing = ROMPTR(dev, P.data[4]);
-                else
-                if (P.version == 2)
-                        timing = ROMPTR(dev, P.data[8]);
-        }
-
-        if (timing && timing[0] == 0x10) {
-                u8 *ramcfg = nouveau_perf_ramcfg(dev, freq, ver, len);
-                if (ramcfg && ramcfg[1] < timing[2]) {
-                        *ver = timing[0];
-                        *len = timing[3];
-                        return timing + timing[1] + (ramcfg[1] * timing[3]);
-                }
-        }
-
-        return NULL;
-}
-
-static void
-legacy_perf_init(struct drm_device *dev)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nvbios *bios = &drm->vbios;
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        char *perf, *entry, *bmp = &bios->data[bios->offset];
-        int headerlen, use_straps;
-
-        if (bmp[5] < 0x5 || bmp[6] < 0x14) {
-                NV_DEBUG(drm, "BMP version too old for perf\n");
-                return;
-        }
-
-        perf = ROMPTR(dev, bmp[0x73]);
-        if (!perf) {
-                NV_DEBUG(drm, "No memclock table pointer found.\n");
-                return;
-        }
-
-        switch (perf[0]) {
-        case 0x12:
-        case 0x14:
-        case 0x18:
-                use_straps = 0;
-                headerlen = 1;
-                break;
-        case 0x01:
-                use_straps = perf[1] & 1;
-                headerlen = (use_straps ? 8 : 2);
-                break;
-        default:
-                NV_WARN(drm, "Unknown memclock table version %x.\n", perf[0]);
-                return;
-        }
-
-        entry = perf + headerlen;
-        if (use_straps)
-                entry += (nv_rd32(device, NV_PEXTDEV_BOOT_0) & 0x3c) >> 1;
-
-        sprintf(pm->perflvl[0].name, "performance_level_0");
-        pm->perflvl[0].memory = ROM16(entry[0]) * 20;
-        pm->nr_perflvl = 1;
-}
-
-static void
-nouveau_perf_voltage(struct drm_device *dev, struct nouveau_pm_level *perflvl)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct bit_entry P;
-        u8 *vmap;
-        int id;
-
-        id = perflvl->volt_min;
-        perflvl->volt_min = 0;
-
-        /* boards using voltage table version <0x40 store the voltage
-         * level directly in the perflvl entry as a multiple of 10mV
-         */
-        if (drm->pm->voltage.version < 0x40) {
-                perflvl->volt_min = id * 10000;
-                perflvl->volt_max = perflvl->volt_min;
-                return;
-        }
-
-        /* on newer ones, the perflvl stores an index into yet another
-         * vbios table containing a min/max voltage value for the perflvl
-         */
-        if (bit_table(dev, 'P', &P) || P.version != 2 || P.length < 34) {
-                NV_DEBUG(drm, "where's our volt map table ptr? %d %d\n",
-                         P.version, P.length);
-                return;
-        }
-
-        vmap = ROMPTR(dev, P.data[32]);
-        if (!vmap) {
-                NV_DEBUG(drm, "volt map table pointer invalid\n");
-                return;
-        }
-
-        if (id < vmap[3]) {
-                vmap += vmap[1] + (vmap[2] * id);
-                perflvl->volt_min = ROM32(vmap[0]);
-                perflvl->volt_max = ROM32(vmap[4]);
-        }
-}
-
-void
-nouveau_perf_init(struct drm_device *dev)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        struct nvbios *bios = &drm->vbios;
-        u8 *perf, ver, hdr, cnt, len;
-        int ret, vid, i = -1;
-
-        if (bios->type == NVBIOS_BMP && bios->data[bios->offset + 6] < 0x25) {
-                legacy_perf_init(dev);
-                return;
-        }
-
-        perf = nouveau_perf_table(dev, &ver);
-
-        while ((perf = nouveau_perf_entry(dev, ++i, &ver, &hdr, &cnt, &len))) {
-                struct nouveau_pm_level *perflvl = &pm->perflvl[pm->nr_perflvl];
-
-                if (perf[0] == 0xff)
-                        continue;
-
-                switch (ver) {
-                case 0x12:
-                case 0x13:
-                case 0x15:
-                        perflvl->fanspeed = perf[55];
-                        if (hdr > 56)
-                                perflvl->volt_min = perf[56];
-                        perflvl->core = ROM32(perf[1]) * 10;
-                        perflvl->memory = ROM32(perf[5]) * 20;
-                        break;
-                case 0x21:
-                case 0x23:
-                case 0x24:
-                        perflvl->fanspeed = perf[4];
-                        perflvl->volt_min = perf[5];
-                        perflvl->shader = ROM16(perf[6]) * 1000;
-                        perflvl->core = perflvl->shader;
-                        perflvl->core += (signed char)perf[8] * 1000;
-                        if (nv_device(drm->device)->chipset == 0x49 ||
-                            nv_device(drm->device)->chipset == 0x4b)
-                                perflvl->memory = ROM16(perf[11]) * 1000;
-                        else
-                                perflvl->memory = ROM16(perf[11]) * 2000;
-                        break;
-                case 0x25:
-                        perflvl->fanspeed = perf[4];
-                        perflvl->volt_min = perf[5];
-                        perflvl->core = ROM16(perf[6]) * 1000;
-                        perflvl->shader = ROM16(perf[10]) * 1000;
-                        perflvl->memory = ROM16(perf[12]) * 1000;
-                        break;
-                case 0x30:
-                        perflvl->memscript = ROM16(perf[2]);
-                case 0x35:
-                        perflvl->fanspeed = perf[6];
-                        perflvl->volt_min = perf[7];
-                        perflvl->core = ROM16(perf[8]) * 1000;
-                        perflvl->shader = ROM16(perf[10]) * 1000;
-                        perflvl->memory = ROM16(perf[12]) * 1000;
-                        perflvl->vdec = ROM16(perf[16]) * 1000;
-                        perflvl->dom6 = ROM16(perf[20]) * 1000;
-                        break;
-                case 0x40:
-#define subent(n) ((ROM16(perf[hdr + (n) * len]) & 0xfff) * 1000)
-                        perflvl->fanspeed = 0; /*XXX*/
-                        perflvl->volt_min = perf[2];
-                        if (nv_device(drm->device)->card_type == NV_50) {
-                                perflvl->core   = subent(0);
-                                perflvl->shader = subent(1);
-                                perflvl->memory = subent(2);
-                                perflvl->vdec   = subent(3);
-                                perflvl->unka0  = subent(4);
-                        } else {
-                                perflvl->hub06  = subent(0);
-                                perflvl->hub01  = subent(1);
-                                perflvl->copy   = subent(2);
-                                perflvl->shader = subent(3);
-                                perflvl->rop    = subent(4);
-                                perflvl->memory = subent(5);
-                                perflvl->vdec   = subent(6);
-                                perflvl->daemon = subent(10);
-                                perflvl->hub07  = subent(11);
-                                perflvl->core   = perflvl->shader / 2;
-                        }
-                        break;
-                }
-
-                /* make sure vid is valid */
-                nouveau_perf_voltage(dev, perflvl);
-                if (pm->voltage.supported && perflvl->volt_min) {
-                        vid = nouveau_volt_vid_lookup(dev, perflvl->volt_min);
-                        if (vid < 0) {
-                                NV_DEBUG(drm, "perflvl %d, bad vid\n", i);
-                                continue;
-                        }
-                }
-
-                /* get the corresponding memory timings */
-                ret = nouveau_mem_timing_calc(dev, perflvl->memory,
-                                                  &perflvl->timing);
-                if (ret) {
-                        NV_DEBUG(drm, "perflvl %d, bad timing: %d\n", i, ret);
-                        continue;
-                }
-
-                snprintf(perflvl->name, sizeof(perflvl->name),
-                         "performance_level_%d", i);
-                perflvl->id = i;
-
-                snprintf(perflvl->profile.name, sizeof(perflvl->profile.name),
-                         "%d", perflvl->id);
-                perflvl->profile.func = &nouveau_pm_static_profile_func;
-                list_add_tail(&perflvl->profile.head, &pm->profiles);
-
-
-                pm->nr_perflvl++;
-        }
-}
-
-void
-nouveau_perf_fini(struct drm_device *dev)
-{
-}
diff --git a/drivers/gpu/drm/nouveau/nouveau_pm.h b/drivers/gpu/drm/nouveau/nouveau_pm.h
deleted file mode 100644
index 73b789c230a9..000000000000
--- a/drivers/gpu/drm/nouveau/nouveau_pm.h
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#ifndef __NOUVEAU_PM_H__
-#define __NOUVEAU_PM_H__
-
-#include <subdev/bios/pll.h>
-#include <subdev/clock.h>
-
-struct nouveau_pm_voltage_level {
-        u32 voltage; /* microvolts */
-        u8  vid;
-};
-
-struct nouveau_pm_voltage {
-        bool supported;
-        u8 version;
-        u8 vid_mask;
-
-        struct nouveau_pm_voltage_level *level;
-        int nr_level;
-};
-
-/* Exclusive upper limits */
-#define NV_MEM_CL_DDR2_MAX 8
-#define NV_MEM_WR_DDR2_MAX 9
-#define NV_MEM_CL_DDR3_MAX 17
-#define NV_MEM_WR_DDR3_MAX 17
-#define NV_MEM_CL_GDDR3_MAX 16
-#define NV_MEM_WR_GDDR3_MAX 18
-#define NV_MEM_CL_GDDR5_MAX 21
-#define NV_MEM_WR_GDDR5_MAX 20
-
-struct nouveau_pm_memtiming {
-        int id;
-
-        u32 reg[9];
-        u32 mr[4];
-
-        u8 tCWL;
-
-        u8 odt;
-        u8 drive_strength;
-};
-
-struct nouveau_pm_tbl_header {
-        u8 version;
-        u8 header_len;
-        u8 entry_cnt;
-        u8 entry_len;
-};
-
-struct nouveau_pm_tbl_entry {
-        u8 tWR;
-        u8 tWTR;
-        u8 tCL;
-        u8 tRC;
-        u8 empty_4;
-        u8 tRFC;        /* Byte 5 */
-        u8 empty_6;
-        u8 tRAS;        /* Byte 7 */
-        u8 empty_8;
-        u8 tRP;         /* Byte 9 */
-        u8 tRCDRD;
-        u8 tRCDWR;
-        u8 tRRD;
-        u8 tUNK_13;
-        u8 RAM_FT1;             /* 14, a bitmask of random RAM features */
-        u8 empty_15;
-        u8 tUNK_16;
-        u8 empty_17;
-        u8 tUNK_18;
-        u8 tCWL;
-        u8 tUNK_20, tUNK_21;
-};
-
-struct nouveau_pm_profile;
-struct nouveau_pm_profile_func {
-        void (*destroy)(struct nouveau_pm_profile *);
-        void (*init)(struct nouveau_pm_profile *);
-        void (*fini)(struct nouveau_pm_profile *);
-        struct nouveau_pm_level *(*select)(struct nouveau_pm_profile *);
-};
-
-struct nouveau_pm_profile {
-        const struct nouveau_pm_profile_func *func;
-        struct list_head head;
-        char name[8];
-};
-
-#define NOUVEAU_PM_MAX_LEVEL 8
-struct nouveau_pm_level {
-        struct nouveau_pm_profile profile;
-        struct device_attribute dev_attr;
-        char name[32];
-        int id;
-
-        struct nouveau_pm_memtiming timing;
-        u32 memory;
-        u16 memscript;
-
-        u32 core;
-        u32 shader;
-        u32 rop;
-        u32 copy;
-        u32 daemon;
-        u32 vdec;
-        u32 dom6;
-        u32 unka0;      /* nva3:nvc0 */
-        u32 hub01;      /* nvc0- */
-        u32 hub06;      /* nvc0- */
-        u32 hub07;      /* nvc0- */
-
-        u32 volt_min; /* microvolts */
-        u32 volt_max;
-        u8  fanspeed;
-};
-
-struct nouveau_pm_temp_sensor_constants {
-        u16 offset_constant;
-        s16 offset_mult;
-        s16 offset_div;
-        s16 slope_mult;
-        s16 slope_div;
-};
-
-struct nouveau_pm_threshold_temp {
-        s16 critical;
-        s16 down_clock;
-};
-
-struct nouveau_pm {
-        struct drm_device *dev;
-
-        struct nouveau_pm_voltage voltage;
-        struct nouveau_pm_level perflvl[NOUVEAU_PM_MAX_LEVEL];
-        int nr_perflvl;
-        struct nouveau_pm_temp_sensor_constants sensor_constants;
-        struct nouveau_pm_threshold_temp threshold_temp;
-
-        struct nouveau_pm_profile *profile_ac;
-        struct nouveau_pm_profile *profile_dc;
-        struct nouveau_pm_profile *profile;
-        struct list_head profiles;
-
-        struct nouveau_pm_level boot;
-        struct nouveau_pm_level *cur;
-
-        struct device *hwmon;
-        struct notifier_block acpi_nb;
-
-        int  (*clocks_get)(struct drm_device *, struct nouveau_pm_level *);
-        void *(*clocks_pre)(struct drm_device *, struct nouveau_pm_level *);
-        int (*clocks_set)(struct drm_device *, void *);
-
-        int (*voltage_get)(struct drm_device *);
-        int (*voltage_set)(struct drm_device *, int voltage);
-};
-
-static inline struct nouveau_pm *
-nouveau_pm(struct drm_device *dev)
-{
-        return nouveau_drm(dev)->pm;
-}
-
-struct nouveau_mem_exec_func {
-        struct drm_device *dev;
-        void (*precharge)(struct nouveau_mem_exec_func *);
-        void (*refresh)(struct nouveau_mem_exec_func *);
-        void (*refresh_auto)(struct nouveau_mem_exec_func *, bool);
-        void (*refresh_self)(struct nouveau_mem_exec_func *, bool);
-        void (*wait)(struct nouveau_mem_exec_func *, u32 nsec);
-        u32  (*mrg)(struct nouveau_mem_exec_func *, int mr);
-        void (*mrs)(struct nouveau_mem_exec_func *, int mr, u32 data);
-        void (*clock_set)(struct nouveau_mem_exec_func *);
-        void (*timing_set)(struct nouveau_mem_exec_func *);
-        void *priv;
-};
-
-/* nouveau_mem.c */
-int  nouveau_mem_exec(struct nouveau_mem_exec_func *,
-                      struct nouveau_pm_level *);
-
-/* nouveau_pm.c */
-int  nouveau_pm_init(struct drm_device *dev);
-void nouveau_pm_fini(struct drm_device *dev);
-void nouveau_pm_resume(struct drm_device *dev);
-extern const struct nouveau_pm_profile_func nouveau_pm_static_profile_func;
-void nouveau_pm_trigger(struct drm_device *dev);
-
-/* nouveau_volt.c */
-void nouveau_volt_init(struct drm_device *);
-void nouveau_volt_fini(struct drm_device *);
-int  nouveau_volt_vid_lookup(struct drm_device *, int voltage);
-int  nouveau_volt_lvl_lookup(struct drm_device *, int vid);
-int  nouveau_voltage_gpio_get(struct drm_device *);
-int  nouveau_voltage_gpio_set(struct drm_device *, int voltage);
-
-/* nouveau_perf.c */
-void nouveau_perf_init(struct drm_device *);
-void nouveau_perf_fini(struct drm_device *);
-u8 *nouveau_perf_rammap(struct drm_device *, u32 freq, u8 *ver,
-                        u8 *hdr, u8 *cnt, u8 *len);
-u8 *nouveau_perf_ramcfg(struct drm_device *, u32 freq, u8 *ver, u8 *len);
-u8 *nouveau_perf_timing(struct drm_device *, u32 freq, u8 *ver, u8 *len);
-
-/* nouveau_mem.c */
-void nouveau_mem_timing_init(struct drm_device *);
-void nouveau_mem_timing_fini(struct drm_device *);
-
-/* nv04_pm.c */
-int nv04_pm_clocks_get(struct drm_device *, struct nouveau_pm_level *);
-void *nv04_pm_clocks_pre(struct drm_device *, struct nouveau_pm_level *);
-int nv04_pm_clocks_set(struct drm_device *, void *);
-
-/* nv40_pm.c */
-int nv40_pm_clocks_get(struct drm_device *, struct nouveau_pm_level *);
-void *nv40_pm_clocks_pre(struct drm_device *, struct nouveau_pm_level *);
-int nv40_pm_clocks_set(struct drm_device *, void *);
-int nv40_pm_pwm_get(struct drm_device *, int, u32 *, u32 *);
-int nv40_pm_pwm_set(struct drm_device *, int, u32, u32);
-
-/* nv50_pm.c */
-int nv50_pm_clocks_get(struct drm_device *, struct nouveau_pm_level *);
-void *nv50_pm_clocks_pre(struct drm_device *, struct nouveau_pm_level *);
-int nv50_pm_clocks_set(struct drm_device *, void *);
-int nv50_pm_pwm_get(struct drm_device *, int, u32 *, u32 *);
-int nv50_pm_pwm_set(struct drm_device *, int, u32, u32);
-
-/* nva3_pm.c */
-int nva3_pm_clocks_get(struct drm_device *, struct nouveau_pm_level *);
-void *nva3_pm_clocks_pre(struct drm_device *, struct nouveau_pm_level *);
-int nva3_pm_clocks_set(struct drm_device *, void *);
-
-/* nvc0_pm.c */
-int nvc0_pm_clocks_get(struct drm_device *, struct nouveau_pm_level *);
-void *nvc0_pm_clocks_pre(struct drm_device *, struct nouveau_pm_level *);
-int nvc0_pm_clocks_set(struct drm_device *, void *);
-
-/* nouveau_mem.c */
-int  nouveau_mem_timing_calc(struct drm_device *, u32 freq,
-                             struct nouveau_pm_memtiming *);
-void nouveau_mem_timing_read(struct drm_device *,
-                             struct nouveau_pm_memtiming *);
-
-static inline int
-nva3_calc_pll(struct drm_device *dev, struct nvbios_pll *pll, u32 freq,
-              int *N, int *fN, int *M, int *P)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_clock *clk = nouveau_clock(device);
-        struct nouveau_pll_vals pv;
-        int ret;
-
-        ret = clk->pll_calc(clk, pll, freq, &pv);
-        *N = pv.N1;
-        *M = pv.M1;
-        *P = pv.log2P;
-        return ret;
-}
-
-#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_sysfs.c b/drivers/gpu/drm/nouveau/nouveau_sysfs.c
new file mode 100644
index 000000000000..89201a17ce75
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_sysfs.c
@@ -0,0 +1,162 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include "nouveau_sysfs.h"
+
+#include <core/object.h>
+#include <core/class.h>
+
+static inline struct drm_device *
+drm_device(struct device *d)
+{
+        return pci_get_drvdata(to_pci_dev(d));
+}
+
+#define snappendf(p,r,f,a...) do {                                             \
+        snprintf(p, r, f, ##a);                                                \
+        r -= strlen(p);                                                        \
+        p += strlen(p);                                                        \
+} while(0)
+
+static ssize_t
+nouveau_sysfs_pstate_get(struct device *d, struct device_attribute *a, char *b)
+{
+        struct nouveau_sysfs *sysfs = nouveau_sysfs(drm_device(d));
+        struct nv_control_pstate_info info;
+        size_t cnt = PAGE_SIZE;
+        char *buf = b;
+        int ret, i;
+
+        ret = nv_exec(sysfs->ctrl, NV_CONTROL_PSTATE_INFO, &info, sizeof(info));
+        if (ret)
+                return ret;
+
+        for (i = 0; i < info.count + 1; i++) {
+                const s32 state = i < info.count ? i :
+                        NV_CONTROL_PSTATE_ATTR_STATE_CURRENT;
+                struct nv_control_pstate_attr attr = {
+                        .state = state,
+                        .index = 0,
+                };
+
+                ret = nv_exec(sysfs->ctrl, NV_CONTROL_PSTATE_ATTR,
+                             &attr, sizeof(attr));
+                if (ret)
+                        return ret;
+
+                if (i < info.count)
+                        snappendf(buf, cnt, "%02x:", attr.state);
+                else
+                        snappendf(buf, cnt, "--:");
+
+                attr.index = 0;
+                do {
+                        attr.state = state;
+                        ret = nv_exec(sysfs->ctrl, NV_CONTROL_PSTATE_ATTR,
+                                     &attr, sizeof(attr));
+                        if (ret)
+                                return ret;
+
+                        snappendf(buf, cnt, " %s %d", attr.name, attr.min);
+                        if (attr.min != attr.max)
+                                snappendf(buf, cnt, "-%d", attr.max);
+                        snappendf(buf, cnt, " %s", attr.unit);
+                } while (attr.index);
+
+                if ((state >= 0 && info.pstate == state) ||
+                    (state <  0 && info.ustate < 0))
+                        snappendf(buf, cnt, " *");
+                snappendf(buf, cnt, "\n");
+        }
+
+        return strlen(b);
+}
+
+static ssize_t
+nouveau_sysfs_pstate_set(struct device *d, struct device_attribute *a,
+                         const char *buf, size_t count)
+{
+        struct nouveau_sysfs *sysfs = nouveau_sysfs(drm_device(d));
+        struct nv_control_pstate_user args;
+        long value, ret;
+        char *tmp;
+
+        if ((tmp = strchr(buf, '\n')))
+                *tmp = '\0';
+
+        if (!strcasecmp(buf, "none"))
+                args.state = NV_CONTROL_PSTATE_USER_STATE_UNKNOWN;
+        else
+        if (!strcasecmp(buf, "auto"))
+                args.state = NV_CONTROL_PSTATE_USER_STATE_PERFMON;
+        else {
+                ret = kstrtol(buf, 16, &value);
+                if (ret)
+                        return ret;
+                args.state = value;
+        }
+
+        ret = nv_exec(sysfs->ctrl, NV_CONTROL_PSTATE_USER, &args, sizeof(args));
+        if (ret < 0)
+                return ret;
+
+        return count;
+}
+
+static DEVICE_ATTR(pstate, S_IRUGO | S_IWUSR,
+                   nouveau_sysfs_pstate_get, nouveau_sysfs_pstate_set);
+
+void
+nouveau_sysfs_fini(struct drm_device *dev)
+{
+        struct nouveau_sysfs *sysfs = nouveau_sysfs(dev);
+        struct nouveau_drm *drm = nouveau_drm(dev);
+
+        if (sysfs->ctrl) {
+                device_remove_file(&dev->pdev->dev, &dev_attr_pstate);
+                nouveau_object_del(nv_object(drm), NVDRM_DEVICE, NVDRM_CONTROL);
+        }
+
+        drm->sysfs = NULL;
+        kfree(sysfs);
+}
+
+int
+nouveau_sysfs_init(struct drm_device *dev)
+{
+        struct nouveau_drm *drm = nouveau_drm(dev);
+        struct nouveau_sysfs *sysfs;
+        int ret;
+
+        sysfs = drm->sysfs = kzalloc(sizeof(*sysfs), GFP_KERNEL);
+        if (!sysfs)
+                return -ENOMEM;
+
+        ret = nouveau_object_new(nv_object(drm), NVDRM_DEVICE, NVDRM_CONTROL,
+                                 NV_CONTROL_CLASS, NULL, 0, &sysfs->ctrl);
+        if (ret == 0)
+                device_create_file(&dev->pdev->dev, &dev_attr_pstate);
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nouveau_sysfs.h b/drivers/gpu/drm/nouveau/nouveau_sysfs.h
new file mode 100644
index 000000000000..74b47f1e01ed
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nouveau_sysfs.h
@@ -0,0 +1,19 @@
+#ifndef __NOUVEAU_SYSFS_H__
+#define __NOUVEAU_SYSFS_H__
+
+#include "nouveau_drm.h"
+
+struct nouveau_sysfs {
+        struct nouveau_object *ctrl;
+};
+
+static inline struct nouveau_sysfs *
+nouveau_sysfs(struct drm_device *dev)
+{
+        return nouveau_drm(dev)->sysfs;
+}
+
+int  nouveau_sysfs_init(struct drm_device *);
+void nouveau_sysfs_fini(struct drm_device *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/nouveau_volt.c b/drivers/gpu/drm/nouveau/nouveau_volt.c
deleted file mode 100644
index 9976414cbe50..000000000000
--- a/drivers/gpu/drm/nouveau/nouveau_volt.c
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include <drm/drmP.h>
-
-#include "nouveau_drm.h"
-#include "nouveau_pm.h"
-
-#include <subdev/bios/gpio.h>
-#include <subdev/gpio.h>
-
-static const enum dcb_gpio_func_name vidtag[] = { 0x04, 0x05, 0x06, 0x1a, 0x73 };
-static int nr_vidtag = sizeof(vidtag) / sizeof(vidtag[0]);
-
-int
-nouveau_voltage_gpio_get(struct drm_device *dev)
-{
-        struct nouveau_pm_voltage *volt = &nouveau_pm(dev)->voltage;
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_gpio *gpio = nouveau_gpio(device);
-        u8 vid = 0;
-        int i;
-
-        for (i = 0; i < nr_vidtag; i++) {
-                if (!(volt->vid_mask & (1 << i)))
-                        continue;
-
-                vid |= gpio->get(gpio, 0, vidtag[i], 0xff) << i;
-        }
-
-        return nouveau_volt_lvl_lookup(dev, vid);
-}
-
-int
-nouveau_voltage_gpio_set(struct drm_device *dev, int voltage)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_gpio *gpio = nouveau_gpio(device);
-        struct nouveau_pm_voltage *volt = &nouveau_pm(dev)->voltage;
-        int vid, i;
-
-        vid = nouveau_volt_vid_lookup(dev, voltage);
-        if (vid < 0)
-                return vid;
-
-        for (i = 0; i < nr_vidtag; i++) {
-                if (!(volt->vid_mask & (1 << i)))
-                        continue;
-
-                gpio->set(gpio, 0, vidtag[i], 0xff, !!(vid & (1 << i)));
-        }
-
-        return 0;
-}
-
-int
-nouveau_volt_vid_lookup(struct drm_device *dev, int voltage)
-{
-        struct nouveau_pm_voltage *volt = &nouveau_pm(dev)->voltage;
-        int i;
-
-        for (i = 0; i < volt->nr_level; i++) {
-                if (volt->level[i].voltage == voltage)
-                        return volt->level[i].vid;
-        }
-
-        return -ENOENT;
-}
-
-int
-nouveau_volt_lvl_lookup(struct drm_device *dev, int vid)
-{
-        struct nouveau_pm_voltage *volt = &nouveau_pm(dev)->voltage;
-        int i;
-
-        for (i = 0; i < volt->nr_level; i++) {
-                if (volt->level[i].vid == vid)
-                        return volt->level[i].voltage;
-        }
-
-        return -ENOENT;
-}
-
-void
-nouveau_volt_init(struct drm_device *dev)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_gpio *gpio = nouveau_gpio(drm->device);
-        struct nouveau_pm *pm = nouveau_pm(dev);
-        struct nouveau_pm_voltage *voltage = &pm->voltage;
-        struct nvbios *bios = &drm->vbios;
-        struct dcb_gpio_func func;
-        struct bit_entry P;
-        u8 *volt = NULL, *entry;
-        int i, headerlen, recordlen, entries, vidmask, vidshift;
-
-        if (bios->type == NVBIOS_BIT) {
-                if (bit_table(dev, 'P', &P))
-                        return;
-
-                if (P.version == 1)
-                        volt = ROMPTR(dev, P.data[16]);
-                else
-                if (P.version == 2)
-                        volt = ROMPTR(dev, P.data[12]);
-                else {
-                        NV_WARN(drm, "unknown volt for BIT P %d\n", P.version);
-                }
-        } else {
-                if (bios->data[bios->offset + 6] < 0x27) {
-                        NV_DEBUG(drm, "BMP version too old for voltage\n");
-                        return;
-                }
-
-                volt = ROMPTR(dev, bios->data[bios->offset + 0x98]);
-        }
-
-        if (!volt) {
-                NV_DEBUG(drm, "voltage table pointer invalid\n");
-                return;
-        }
-
-        switch (volt[0]) {
-        case 0x10:
-        case 0x11:
-        case 0x12:
-                headerlen = 5;
-                recordlen = volt[1];
-                entries   = volt[2];
-                vidshift  = 0;
-                vidmask   = volt[4];
-                break;
-        case 0x20:
-                headerlen = volt[1];
-                recordlen = volt[3];
-                entries   = volt[2];
-                vidshift  = 0; /* could be vidshift like 0x30? */
-                vidmask   = volt[5];
-                break;
-        case 0x30:
-                headerlen = volt[1];
-                recordlen = volt[2];
-                entries   = volt[3];
-                vidmask   = volt[4];
-                /* no longer certain what volt[5] is, if it's related to
-                 * the vid shift then it's definitely not a function of
-                 * how many bits are set.
-                 *
-                 * after looking at a number of nva3+ vbios images, they
-                 * all seem likely to have a static shift of 2.. lets
-                 * go with that for now until proven otherwise.
-                 */
-                vidshift  = 2;
-                break;
-        case 0x40:
-                headerlen = volt[1];
-                recordlen = volt[2];
-                entries   = volt[3]; /* not a clue what the entries are for.. */
-                vidmask   = volt[11]; /* guess.. */
-                vidshift  = 0;
-                break;
-        default:
-                NV_WARN(drm, "voltage table 0x%02x unknown\n", volt[0]);
-                return;
-        }
-
-        /* validate vid mask */
-        voltage->vid_mask = vidmask;
-        if (!voltage->vid_mask)
-                return;
-
-        i = 0;
-        while (vidmask) {
-                if (i > nr_vidtag) {
-                        NV_DEBUG(drm, "vid bit %d unknown\n", i);
-                        return;
-                }
-
-                if (gpio && gpio->find(gpio, 0, vidtag[i], 0xff, &func)) {
-                        NV_DEBUG(drm, "vid bit %d has no gpio tag\n", i);
-                        return;
-                }
-
-                vidmask >>= 1;
-                i++;
-        }
-
-        /* parse vbios entries into common format */
-        voltage->version = volt[0];
-        if (voltage->version < 0x40) {
-                voltage->nr_level = entries;
-                voltage->level =
-                        kcalloc(entries, sizeof(*voltage->level), GFP_KERNEL);
-                if (!voltage->level)
-                        return;
-
-                entry = volt + headerlen;
-                for (i = 0; i < entries; i++, entry += recordlen) {
-                        voltage->level[i].voltage = entry[0] * 10000;
-                        voltage->level[i].vid     = entry[1] >> vidshift;
-                }
-        } else {
-                u32 volt_uv = ROM32(volt[4]);
-                s16 step_uv = ROM16(volt[8]);
-                u8 vid;
-
-                voltage->nr_level = voltage->vid_mask + 1;
-                voltage->level = kcalloc(voltage->nr_level,
-                                         sizeof(*voltage->level), GFP_KERNEL);
-                if (!voltage->level)
-                        return;
-
-                for (vid = 0; vid <= voltage->vid_mask; vid++) {
-                        voltage->level[vid].voltage = volt_uv;
-                        voltage->level[vid].vid = vid;
-                        volt_uv += step_uv;
-                }
-        }
-
-        voltage->supported = true;
-}
-
-void
-nouveau_volt_fini(struct drm_device *dev)
-{
-        struct nouveau_pm_voltage *volt = &nouveau_pm(dev)->voltage;
-
-        kfree(volt->level);
-}
diff --git a/drivers/gpu/drm/nouveau/nv04_pm.c b/drivers/gpu/drm/nouveau/nv04_pm.c
deleted file mode 100644
index 27afc0ea28b0..000000000000
--- a/drivers/gpu/drm/nouveau/nv04_pm.c
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include <drm/drmP.h>
-#include "nouveau_drm.h"
-#include "nouveau_reg.h"
-#include "dispnv04/hw.h"
-#include "nouveau_pm.h"
-
-#include <subdev/bios/pll.h>
-#include <subdev/clock.h>
-#include <subdev/timer.h>
-
-int
-nv04_pm_clocks_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
-{
-        int ret;
-
-        ret = nouveau_hw_get_clock(dev, PLL_CORE);
-        if (ret < 0)
-                return ret;
-        perflvl->core = ret;
-
-        ret = nouveau_hw_get_clock(dev, PLL_MEMORY);
-        if (ret < 0)
-                return ret;
-        perflvl->memory = ret;
-
-        return 0;
-}
-
-struct nv04_pm_clock {
-        struct nvbios_pll pll;
-        struct nouveau_pll_vals calc;
-};
-
-struct nv04_pm_state {
-        struct nv04_pm_clock core;
-        struct nv04_pm_clock memory;
-};
-
-static int
-calc_pll(struct drm_device *dev, u32 id, int khz, struct nv04_pm_clock *clk)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_bios *bios = nouveau_bios(device);
-        struct nouveau_clock *pclk = nouveau_clock(device);
-        int ret;
-
-        ret = nvbios_pll_parse(bios, id, &clk->pll);
-        if (ret)
-                return ret;
-
-        ret = pclk->pll_calc(pclk, &clk->pll, khz, &clk->calc);
-        if (!ret)
-                return -EINVAL;
-
-        return 0;
-}
-
-void *
-nv04_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
-{
-        struct nv04_pm_state *info;
-        int ret;
-
-        info = kzalloc(sizeof(*info), GFP_KERNEL);
-        if (!info)
-                return ERR_PTR(-ENOMEM);
-
-        ret = calc_pll(dev, PLL_CORE, perflvl->core, &info->core);
-        if (ret)
-                goto error;
-
-        if (perflvl->memory) {
-                ret = calc_pll(dev, PLL_MEMORY, perflvl->memory, &info->memory);
-                if (ret)
-                        goto error;
-        }
-
-        return info;
-error:
-        kfree(info);
-        return ERR_PTR(ret);
-}
-
-static void
-prog_pll(struct drm_device *dev, struct nv04_pm_clock *clk)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_clock *pclk = nouveau_clock(device);
-        u32 reg = clk->pll.reg;
-
-        /* thank the insane nouveau_hw_setpll() interface for this */
-        if (device->card_type >= NV_40)
-                reg += 4;
-
-        pclk->pll_prog(pclk, reg, &clk->calc);
-}
-
-int
-nv04_pm_clocks_set(struct drm_device *dev, void *pre_state)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_timer *ptimer = nouveau_timer(device);
-        struct nv04_pm_state *state = pre_state;
-
-        prog_pll(dev, &state->core);
-
-        if (state->memory.pll.reg) {
-                prog_pll(dev, &state->memory);
-                if (device->card_type < NV_30) {
-                        if (device->card_type == NV_20)
-                                nv_mask(device, 0x1002c4, 0, 1 << 20);
-
-                        /* Reset the DLLs */
-                        nv_mask(device, 0x1002c0, 0, 1 << 8);
-                }
-        }
-
-        nv_ofuncs(ptimer)->init(nv_object(ptimer));
-
-        kfree(state);
-        return 0;
-}
diff --git a/drivers/gpu/drm/nouveau/nv40_pm.c b/drivers/gpu/drm/nouveau/nv40_pm.c
deleted file mode 100644
index 625f80d53dc2..000000000000
--- a/drivers/gpu/drm/nouveau/nv40_pm.c
+++ /dev/null
@@ -1,353 +0,0 @@
-/*
- * Copyright 2011 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include <drm/drmP.h>
-#include "nouveau_drm.h"
-#include "nouveau_bios.h"
-#include "nouveau_pm.h"
-#include "dispnv04/hw.h"
-
-#include <subdev/bios/pll.h>
-#include <subdev/clock.h>
-#include <subdev/timer.h>
-
-#include <engine/fifo.h>
-
-#define min2(a,b) ((a) < (b) ? (a) : (b))
-
-static u32
-read_pll_1(struct drm_device *dev, u32 reg)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        u32 ctrl = nv_rd32(device, reg + 0x00);
-        int P = (ctrl & 0x00070000) >> 16;
-        int N = (ctrl & 0x0000ff00) >> 8;
-        int M = (ctrl & 0x000000ff) >> 0;
-        u32 ref = 27000, clk = 0;
-
-        if (ctrl & 0x80000000)
-                clk = ref * N / M;
-
-        return clk >> P;
-}
-
-static u32
-read_pll_2(struct drm_device *dev, u32 reg)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        u32 ctrl = nv_rd32(device, reg + 0x00);
-        u32 coef = nv_rd32(device, reg + 0x04);
-        int N2 = (coef & 0xff000000) >> 24;
-        int M2 = (coef & 0x00ff0000) >> 16;
-        int N1 = (coef & 0x0000ff00) >> 8;
-        int M1 = (coef & 0x000000ff) >> 0;
-        int P = (ctrl & 0x00070000) >> 16;
-        u32 ref = 27000, clk = 0;
-
-        if ((ctrl & 0x80000000) && M1) {
-                clk = ref * N1 / M1;
-                if ((ctrl & 0x40000100) == 0x40000000) {
-                        if (M2)
-                                clk = clk * N2 / M2;
-                        else
-                                clk = 0;
-                }
-        }
-
-        return clk >> P;
-}
-
-static u32
-read_clk(struct drm_device *dev, u32 src)
-{
-        switch (src) {
-        case 3:
-                return read_pll_2(dev, 0x004000);
-        case 2:
-                return read_pll_1(dev, 0x004008);
-        default:
-                break;
-        }
-
-        return 0;
-}
-
-int
-nv40_pm_clocks_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        u32 ctrl = nv_rd32(device, 0x00c040);
-
-        perflvl->core   = read_clk(dev, (ctrl & 0x00000003) >> 0);
-        perflvl->shader = read_clk(dev, (ctrl & 0x00000030) >> 4);
-        perflvl->memory = read_pll_2(dev, 0x4020);
-        return 0;
-}
-
-struct nv40_pm_state {
-        u32 ctrl;
-        u32 npll_ctrl;
-        u32 npll_coef;
-        u32 spll;
-        u32 mpll_ctrl;
-        u32 mpll_coef;
-};
-
-static int
-nv40_calc_pll(struct drm_device *dev, u32 reg, struct nvbios_pll *pll,
-              u32 clk, int *N1, int *M1, int *N2, int *M2, int *log2P)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_bios *bios = nouveau_bios(device);
-        struct nouveau_clock *pclk = nouveau_clock(device);
-        struct nouveau_pll_vals coef;
-        int ret;
-
-        ret = nvbios_pll_parse(bios, reg, pll);
-        if (ret)
-                return ret;
-
-        if (clk < pll->vco1.max_freq)
-                pll->vco2.max_freq = 0;
-
-        ret = pclk->pll_calc(pclk, pll, clk, &coef);
-        if (ret == 0)
-                return -ERANGE;
-
-        *N1 = coef.N1;
-        *M1 = coef.M1;
-        if (N2 && M2) {
-                if (pll->vco2.max_freq) {
-                        *N2 = coef.N2;
-                        *M2 = coef.M2;
-                } else {
-                        *N2 = 1;
-                        *M2 = 1;
-                }
-        }
-        *log2P = coef.log2P;
-        return 0;
-}
-
-void *
-nv40_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
-{
-        struct nv40_pm_state *info;
-        struct nvbios_pll pll;
-        int N1, N2, M1, M2, log2P;
-        int ret;
-
-        info = kmalloc(sizeof(*info), GFP_KERNEL);
-        if (!info)
-                return ERR_PTR(-ENOMEM);
-
-        /* core/geometric clock */
-        ret = nv40_calc_pll(dev, 0x004000, &pll, perflvl->core,
-                            &N1, &M1, &N2, &M2, &log2P);
-        if (ret < 0)
-                goto out;
-
-        if (N2 == M2) {
-                info->npll_ctrl = 0x80000100 | (log2P << 16);
-                info->npll_coef = (N1 << 8) | M1;
-        } else {
-                info->npll_ctrl = 0xc0000000 | (log2P << 16);
-                info->npll_coef = (N2 << 24) | (M2 << 16) | (N1 << 8) | M1;
-        }
-
-        /* use the second PLL for shader/rop clock, if it differs from core */
-        if (perflvl->shader && perflvl->shader != perflvl->core) {
-                ret = nv40_calc_pll(dev, 0x004008, &pll, perflvl->shader,
-                                    &N1, &M1, NULL, NULL, &log2P);
-                if (ret < 0)
-                        goto out;
-
-                info->spll = 0xc0000000 | (log2P << 16) | (N1 << 8) | M1;
-                info->ctrl = 0x00000223;
-        } else {
-                info->spll = 0x00000000;
-                info->ctrl = 0x00000333;
-        }
-
-        /* memory clock */
-        if (!perflvl->memory) {
-                info->mpll_ctrl = 0x00000000;
-                goto out;
-        }
-
-        ret = nv40_calc_pll(dev, 0x004020, &pll, perflvl->memory,
-                            &N1, &M1, &N2, &M2, &log2P);
-        if (ret < 0)
-                goto out;
-
-        info->mpll_ctrl  = 0x80000000 | (log2P << 16);
-        info->mpll_ctrl |= min2(pll.bias_p + log2P, pll.max_p) << 20;
-        if (N2 == M2) {
-                info->mpll_ctrl |= 0x00000100;
-                info->mpll_coef  = (N1 << 8) | M1;
-        } else {
-                info->mpll_ctrl |= 0x40000000;
-                info->mpll_coef  = (N2 << 24) | (M2 << 16) | (N1 << 8) | M1;
-        }
-
-out:
-        if (ret < 0) {
-                kfree(info);
-                info = ERR_PTR(ret);
-        }
-        return info;
-}
-
-static bool
-nv40_pm_gr_idle(void *data)
-{
-        struct drm_device *dev = data;
-        struct nouveau_device *device = nouveau_dev(dev);
-
-        if ((nv_rd32(device, 0x400760) & 0x000000f0) >> 4 !=
-            (nv_rd32(device, 0x400760) & 0x0000000f))
-                return false;
-
-        if (nv_rd32(device, 0x400700))
-                return false;
-
-        return true;
-}
-
-int
-nv40_pm_clocks_set(struct drm_device *dev, void *pre_state)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_fifo *pfifo = nouveau_fifo(device);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nv40_pm_state *info = pre_state;
-        unsigned long flags;
-        struct bit_entry M;
-        u32 crtc_mask = 0;
-        u8 sr1[2];
-        int i, ret = -EAGAIN;
-
-        /* determine which CRTCs are active, fetch VGA_SR1 for each */
-        for (i = 0; i < 2; i++) {
-                u32 vbl = nv_rd32(device, 0x600808 + (i * 0x2000));
-                u32 cnt = 0;
-                do {
-                        if (vbl != nv_rd32(device, 0x600808 + (i * 0x2000))) {
-                                nv_wr08(device, 0x0c03c4 + (i * 0x2000), 0x01);
-                                sr1[i] = nv_rd08(device, 0x0c03c5 + (i * 0x2000));
-                                if (!(sr1[i] & 0x20))
-                                        crtc_mask |= (1 << i);
-                                break;
-                        }
-                        udelay(1);
-                } while (cnt++ < 32);
-        }
-
-        /* halt and idle engines */
-        pfifo->pause(pfifo, &flags);
-
-        if (!nv_wait_cb(device, nv40_pm_gr_idle, dev))
-                goto resume;
-
-        ret = 0;
-
-        /* set engine clocks */
-        nv_mask(device, 0x00c040, 0x00000333, 0x00000000);
-        nv_wr32(device, 0x004004, info->npll_coef);
-        nv_mask(device, 0x004000, 0xc0070100, info->npll_ctrl);
-        nv_mask(device, 0x004008, 0xc007ffff, info->spll);
-        mdelay(5);
-        nv_mask(device, 0x00c040, 0x00000333, info->ctrl);
-
-        if (!info->mpll_ctrl)
-                goto resume;
-
-        /* wait for vblank start on active crtcs, disable memory access */
-        for (i = 0; i < 2; i++) {
-                if (!(crtc_mask & (1 << i)))
-                        continue;
-                nv_wait(device, 0x600808 + (i * 0x2000), 0x00010000, 0x00000000);
-                nv_wait(device, 0x600808 + (i * 0x2000), 0x00010000, 0x00010000);
-                nv_wr08(device, 0x0c03c4 + (i * 0x2000), 0x01);
-                nv_wr08(device, 0x0c03c5 + (i * 0x2000), sr1[i] | 0x20);
-        }
-
-        /* prepare ram for reclocking */
-        nv_wr32(device, 0x1002d4, 0x00000001); /* precharge */
-        nv_wr32(device, 0x1002d0, 0x00000001); /* refresh */
-        nv_wr32(device, 0x1002d0, 0x00000001); /* refresh */
-        nv_mask(device, 0x100210, 0x80000000, 0x00000000); /* no auto refresh */
-        nv_wr32(device, 0x1002dc, 0x00000001); /* enable self-refresh */
-
-        /* change the PLL of each memory partition */
-        nv_mask(device, 0x00c040, 0x0000c000, 0x00000000);
-        switch (nv_device(drm->device)->chipset) {
-        case 0x40:
-        case 0x45:
-        case 0x41:
-        case 0x42:
-        case 0x47:
-                nv_mask(device, 0x004044, 0xc0771100, info->mpll_ctrl);
-                nv_mask(device, 0x00402c, 0xc0771100, info->mpll_ctrl);
-                nv_wr32(device, 0x004048, info->mpll_coef);
-                nv_wr32(device, 0x004030, info->mpll_coef);
-        case 0x43:
-        case 0x49:
-        case 0x4b:
-                nv_mask(device, 0x004038, 0xc0771100, info->mpll_ctrl);
-                nv_wr32(device, 0x00403c, info->mpll_coef);
-        default:
-                nv_mask(device, 0x004020, 0xc0771100, info->mpll_ctrl);
-                nv_wr32(device, 0x004024, info->mpll_coef);
-                break;
-        }
-        udelay(100);
-        nv_mask(device, 0x00c040, 0x0000c000, 0x0000c000);
-
-        /* re-enable normal operation of memory controller */
-        nv_wr32(device, 0x1002dc, 0x00000000);
-        nv_mask(device, 0x100210, 0x80000000, 0x80000000);
-        udelay(100);
-
-        /* execute memory reset script from vbios */
-        if (!bit_table(dev, 'M', &M))
-                nouveau_bios_run_init_table(dev, ROM16(M.data[0]), NULL, 0);
-
-        /* make sure we're in vblank (hopefully the same one as before), and
-         * then re-enable crtc memory access
-         */
-        for (i = 0; i < 2; i++) {
-                if (!(crtc_mask & (1 << i)))
-                        continue;
-                nv_wait(device, 0x600808 + (i * 0x2000), 0x00010000, 0x00010000);
-                nv_wr08(device, 0x0c03c4 + (i * 0x2000), 0x01);
-                nv_wr08(device, 0x0c03c5 + (i * 0x2000), sr1[i]);
-        }
-
-        /* resume engines */
-resume:
-        pfifo->start(pfifo, &flags);
-        kfree(info);
-        return ret;
-}
diff --git a/drivers/gpu/drm/nouveau/nv50_pm.c b/drivers/gpu/drm/nouveau/nv50_pm.c
deleted file mode 100644
index 4efc33fa73fc..000000000000
--- a/drivers/gpu/drm/nouveau/nv50_pm.c
+++ /dev/null
@@ -1,855 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include <drm/drmP.h>
-#include "nouveau_drm.h"
-#include "nouveau_bios.h"
-#include "dispnv04/hw.h"
-#include "nouveau_pm.h"
-#include "nouveau_hwsq.h"
-
-#include "nv50_display.h"
-
-#include <subdev/bios/pll.h>
-#include <subdev/clock.h>
-#include <subdev/timer.h>
-#include <subdev/fb.h>
-
-enum clk_src {
-        clk_src_crystal,
-        clk_src_href,
-        clk_src_hclk,
-        clk_src_hclkm3,
-        clk_src_hclkm3d2,
-        clk_src_host,
-        clk_src_nvclk,
-        clk_src_sclk,
-        clk_src_mclk,
-        clk_src_vdec,
-        clk_src_dom6
-};
-
-static u32 read_clk(struct drm_device *, enum clk_src);
-
-static u32
-read_div(struct drm_device *dev)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-
-        switch (nv_device(drm->device)->chipset) {
-        case 0x50: /* it exists, but only has bit 31, not the dividers.. */
-        case 0x84:
-        case 0x86:
-        case 0x98:
-        case 0xa0:
-                return nv_rd32(device, 0x004700);
-        case 0x92:
-        case 0x94:
-        case 0x96:
-                return nv_rd32(device, 0x004800);
-        default:
-                return 0x00000000;
-        }
-}
-
-static u32
-read_pll_src(struct drm_device *dev, u32 base)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        u32 coef, ref = read_clk(dev, clk_src_crystal);
-        u32 rsel = nv_rd32(device, 0x00e18c);
-        int P, N, M, id;
-
-        switch (nv_device(drm->device)->chipset) {
-        case 0x50:
-        case 0xa0:
-                switch (base) {
-                case 0x4020:
-                case 0x4028: id = !!(rsel & 0x00000004); break;
-                case 0x4008: id = !!(rsel & 0x00000008); break;
-                case 0x4030: id = 0; break;
-                default:
-                        NV_ERROR(drm, "ref: bad pll 0x%06x\n", base);
-                        return 0;
-                }
-
-                coef = nv_rd32(device, 0x00e81c + (id * 0x0c));
-                ref *=  (coef & 0x01000000) ? 2 : 4;
-                P    =  (coef & 0x00070000) >> 16;
-                N    = ((coef & 0x0000ff00) >> 8) + 1;
-                M    = ((coef & 0x000000ff) >> 0) + 1;
-                break;
-        case 0x84:
-        case 0x86:
-        case 0x92:
-                coef = nv_rd32(device, 0x00e81c);
-                P    = (coef & 0x00070000) >> 16;
-                N    = (coef & 0x0000ff00) >> 8;
-                M    = (coef & 0x000000ff) >> 0;
-                break;
-        case 0x94:
-        case 0x96:
-        case 0x98:
-                rsel = nv_rd32(device, 0x00c050);
-                switch (base) {
-                case 0x4020: rsel = (rsel & 0x00000003) >> 0; break;
-                case 0x4008: rsel = (rsel & 0x0000000c) >> 2; break;
-                case 0x4028: rsel = (rsel & 0x00001800) >> 11; break;
-                case 0x4030: rsel = 3; break;
-                default:
-                        NV_ERROR(drm, "ref: bad pll 0x%06x\n", base);
-                        return 0;
-                }
-
-                switch (rsel) {
-                case 0: id = 1; break;
-                case 1: return read_clk(dev, clk_src_crystal);
-                case 2: return read_clk(dev, clk_src_href);
-                case 3: id = 0; break;
-                }
-
-                coef =  nv_rd32(device, 0x00e81c + (id * 0x28));
-                P    = (nv_rd32(device, 0x00e824 + (id * 0x28)) >> 16) & 7;
-                P   += (coef & 0x00070000) >> 16;
-                N    = (coef & 0x0000ff00) >> 8;
-                M    = (coef & 0x000000ff) >> 0;
-                break;
-        default:
-                BUG_ON(1);
-        }
-
-        if (M)
-                return (ref * N / M) >> P;
-        return 0;
-}
-
-static u32
-read_pll_ref(struct drm_device *dev, u32 base)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        u32 src, mast = nv_rd32(device, 0x00c040);
-
-        switch (base) {
-        case 0x004028:
-                src = !!(mast & 0x00200000);
-                break;
-        case 0x004020:
-                src = !!(mast & 0x00400000);
-                break;
-        case 0x004008:
-                src = !!(mast & 0x00010000);
-                break;
-        case 0x004030:
-                src = !!(mast & 0x02000000);
-                break;
-        case 0x00e810:
-                return read_clk(dev, clk_src_crystal);
-        default:
-                NV_ERROR(drm, "bad pll 0x%06x\n", base);
-                return 0;
-        }
-
-        if (src)
-                return read_clk(dev, clk_src_href);
-        return read_pll_src(dev, base);
-}
-
-static u32
-read_pll(struct drm_device *dev, u32 base)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        u32 mast = nv_rd32(device, 0x00c040);
-        u32 ctrl = nv_rd32(device, base + 0);
-        u32 coef = nv_rd32(device, base + 4);
-        u32 ref = read_pll_ref(dev, base);
-        u32 clk = 0;
-        int N1, N2, M1, M2;
-
-        if (base == 0x004028 && (mast & 0x00100000)) {
-                /* wtf, appears to only disable post-divider on nva0 */
-                if (nv_device(drm->device)->chipset != 0xa0)
-                        return read_clk(dev, clk_src_dom6);
-        }
-
-        N2 = (coef & 0xff000000) >> 24;
-        M2 = (coef & 0x00ff0000) >> 16;
-        N1 = (coef & 0x0000ff00) >> 8;
-        M1 = (coef & 0x000000ff);
-        if ((ctrl & 0x80000000) && M1) {
-                clk = ref * N1 / M1;
-                if ((ctrl & 0x40000100) == 0x40000000) {
-                        if (M2)
-                                clk = clk * N2 / M2;
-                        else
-                                clk = 0;
-                }
-        }
-
-        return clk;
-}
-
-static u32
-read_clk(struct drm_device *dev, enum clk_src src)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        u32 mast = nv_rd32(device, 0x00c040);
-        u32 P = 0;
-
-        switch (src) {
-        case clk_src_crystal:
-                return device->crystal;
-        case clk_src_href:
-                return 100000; /* PCIE reference clock */
-        case clk_src_hclk:
-                return read_clk(dev, clk_src_href) * 27778 / 10000;
-        case clk_src_hclkm3:
-                return read_clk(dev, clk_src_hclk) * 3;
-        case clk_src_hclkm3d2:
-                return read_clk(dev, clk_src_hclk) * 3 / 2;
-        case clk_src_host:
-                switch (mast & 0x30000000) {
-                case 0x00000000: return read_clk(dev, clk_src_href);
-                case 0x10000000: break;
-                case 0x20000000: /* !0x50 */
-                case 0x30000000: return read_clk(dev, clk_src_hclk);
-                }
-                break;
-        case clk_src_nvclk:
-                if (!(mast & 0x00100000))
-                        P = (nv_rd32(device, 0x004028) & 0x00070000) >> 16;
-                switch (mast & 0x00000003) {
-                case 0x00000000: return read_clk(dev, clk_src_crystal) >> P;
-                case 0x00000001: return read_clk(dev, clk_src_dom6);
-                case 0x00000002: return read_pll(dev, 0x004020) >> P;
-                case 0x00000003: return read_pll(dev, 0x004028) >> P;
-                }
-                break;
-        case clk_src_sclk:
-                P = (nv_rd32(device, 0x004020) & 0x00070000) >> 16;
-                switch (mast & 0x00000030) {
-                case 0x00000000:
-                        if (mast & 0x00000080)
-                                return read_clk(dev, clk_src_host) >> P;
-                        return read_clk(dev, clk_src_crystal) >> P;
-                case 0x00000010: break;
-                case 0x00000020: return read_pll(dev, 0x004028) >> P;
-                case 0x00000030: return read_pll(dev, 0x004020) >> P;
-                }
-                break;
-        case clk_src_mclk:
-                P = (nv_rd32(device, 0x004008) & 0x00070000) >> 16;
-                if (nv_rd32(device, 0x004008) & 0x00000200) {
-                        switch (mast & 0x0000c000) {
-                        case 0x00000000:
-                                return read_clk(dev, clk_src_crystal) >> P;
-                        case 0x00008000:
-                        case 0x0000c000:
-                                return read_clk(dev, clk_src_href) >> P;
-                        }
-                } else {
-                        return read_pll(dev, 0x004008) >> P;
-                }
-                break;
-        case clk_src_vdec:
-                P = (read_div(dev) & 0x00000700) >> 8;
-                switch (nv_device(drm->device)->chipset) {
-                case 0x84:
-                case 0x86:
-                case 0x92:
-                case 0x94:
-                case 0x96:
-                case 0xa0:
-                        switch (mast & 0x00000c00) {
-                        case 0x00000000:
-                                if (nv_device(drm->device)->chipset == 0xa0) /* wtf?? */
-                                        return read_clk(dev, clk_src_nvclk) >> P;
-                                return read_clk(dev, clk_src_crystal) >> P;
-                        case 0x00000400:
-                                return 0;
-                        case 0x00000800:
-                                if (mast & 0x01000000)
-                                        return read_pll(dev, 0x004028) >> P;
-                                return read_pll(dev, 0x004030) >> P;
-                        case 0x00000c00:
-                                return read_clk(dev, clk_src_nvclk) >> P;
-                        }
-                        break;
-                case 0x98:
-                        switch (mast & 0x00000c00) {
-                        case 0x00000000:
-                                return read_clk(dev, clk_src_nvclk) >> P;
-                        case 0x00000400:
-                                return 0;
-                        case 0x00000800:
-                                return read_clk(dev, clk_src_hclkm3d2) >> P;
-                        case 0x00000c00:
-                                return read_clk(dev, clk_src_mclk) >> P;
-                        }
-                        break;
-                }
-                break;
-        case clk_src_dom6:
-                switch (nv_device(drm->device)->chipset) {
-                case 0x50:
-                case 0xa0:
-                        return read_pll(dev, 0x00e810) >> 2;
-                case 0x84:
-                case 0x86:
-                case 0x92:
-                case 0x94:
-                case 0x96:
-                case 0x98:
-                        P = (read_div(dev) & 0x00000007) >> 0;
-                        switch (mast & 0x0c000000) {
-                        case 0x00000000: return read_clk(dev, clk_src_href);
-                        case 0x04000000: break;
-                        case 0x08000000: return read_clk(dev, clk_src_hclk);
-                        case 0x0c000000:
-                                return read_clk(dev, clk_src_hclkm3) >> P;
-                        }
-                        break;
-                default:
-                        break;
-                }
-        default:
-                break;
-        }
-
-        NV_DEBUG(drm, "unknown clock source %d 0x%08x\n", src, mast);
-        return 0;
-}
-
-int
-nv50_pm_clocks_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        if (nv_device(drm->device)->chipset == 0xaa ||
-            nv_device(drm->device)->chipset == 0xac)
-                return 0;
-
-        perflvl->core   = read_clk(dev, clk_src_nvclk);
-        perflvl->shader = read_clk(dev, clk_src_sclk);
-        perflvl->memory = read_clk(dev, clk_src_mclk);
-        if (nv_device(drm->device)->chipset != 0x50) {
-                perflvl->vdec = read_clk(dev, clk_src_vdec);
-                perflvl->dom6 = read_clk(dev, clk_src_dom6);
-        }
-
-        return 0;
-}
-
-struct nv50_pm_state {
-        struct nouveau_pm_level *perflvl;
-        struct hwsq_ucode eclk_hwsq;
-        struct hwsq_ucode mclk_hwsq;
-        u32 mscript;
-        u32 mmast;
-        u32 mctrl;
-        u32 mcoef;
-};
-
-static u32
-calc_pll(struct drm_device *dev, u32 reg, struct nvbios_pll *pll,
-         u32 clk, int *N1, int *M1, int *log2P)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_bios *bios = nouveau_bios(device);
-        struct nouveau_clock *pclk = nouveau_clock(device);
-        struct nouveau_pll_vals coef;
-        int ret;
-
-        ret = nvbios_pll_parse(bios, reg, pll);
-        if (ret)
-                return 0;
-
-        pll->vco2.max_freq = 0;
-        pll->refclk = read_pll_ref(dev, reg);
-        if (!pll->refclk)
-                return 0;
-
-        ret = pclk->pll_calc(pclk, pll, clk, &coef);
-        if (ret == 0)
-                return 0;
-
-        *N1 = coef.N1;
-        *M1 = coef.M1;
-        *log2P = coef.log2P;
-        return ret;
-}
-
-static inline u32
-calc_div(u32 src, u32 target, int *div)
-{
-        u32 clk0 = src, clk1 = src;
-        for (*div = 0; *div <= 7; (*div)++) {
-                if (clk0 <= target) {
-                        clk1 = clk0 << (*div ? 1 : 0);
-                        break;
-                }
-                clk0 >>= 1;
-        }
-
-        if (target - clk0 <= clk1 - target)
-                return clk0;
-        (*div)--;
-        return clk1;
-}
-
-static inline u32
-clk_same(u32 a, u32 b)
-{
-        return ((a / 1000) == (b / 1000));
-}
-
-static void
-mclk_precharge(struct nouveau_mem_exec_func *exec)
-{
-        struct nv50_pm_state *info = exec->priv;
-        struct hwsq_ucode *hwsq = &info->mclk_hwsq;
-
-        hwsq_wr32(hwsq, 0x1002d4, 0x00000001);
-}
-
-static void
-mclk_refresh(struct nouveau_mem_exec_func *exec)
-{
-        struct nv50_pm_state *info = exec->priv;
-        struct hwsq_ucode *hwsq = &info->mclk_hwsq;
-
-        hwsq_wr32(hwsq, 0x1002d0, 0x00000001);
-}
-
-static void
-mclk_refresh_auto(struct nouveau_mem_exec_func *exec, bool enable)
-{
-        struct nv50_pm_state *info = exec->priv;
-        struct hwsq_ucode *hwsq = &info->mclk_hwsq;
-
-        hwsq_wr32(hwsq, 0x100210, enable ? 0x80000000 : 0x00000000);
-}
-
-static void
-mclk_refresh_self(struct nouveau_mem_exec_func *exec, bool enable)
-{
-        struct nv50_pm_state *info = exec->priv;
-        struct hwsq_ucode *hwsq = &info->mclk_hwsq;
-
-        hwsq_wr32(hwsq, 0x1002dc, enable ? 0x00000001 : 0x00000000);
-}
-
-static void
-mclk_wait(struct nouveau_mem_exec_func *exec, u32 nsec)
-{
-        struct nv50_pm_state *info = exec->priv;
-        struct hwsq_ucode *hwsq = &info->mclk_hwsq;
-
-        if (nsec > 1000)
-                hwsq_usec(hwsq, (nsec + 500) / 1000);
-}
-
-static u32
-mclk_mrg(struct nouveau_mem_exec_func *exec, int mr)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        if (mr <= 1)
-                return nv_rd32(device, 0x1002c0 + ((mr - 0) * 4));
-        if (mr <= 3)
-                return nv_rd32(device, 0x1002e0 + ((mr - 2) * 4));
-        return 0;
-}
-
-static void
-mclk_mrs(struct nouveau_mem_exec_func *exec, int mr, u32 data)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        struct nouveau_fb *pfb = nouveau_fb(device);
-        struct nv50_pm_state *info = exec->priv;
-        struct hwsq_ucode *hwsq = &info->mclk_hwsq;
-
-        if (mr <= 1) {
-                if (pfb->ram->ranks > 1)
-                        hwsq_wr32(hwsq, 0x1002c8 + ((mr - 0) * 4), data);
-                hwsq_wr32(hwsq, 0x1002c0 + ((mr - 0) * 4), data);
-        } else
-        if (mr <= 3) {
-                if (pfb->ram->ranks > 1)
-                        hwsq_wr32(hwsq, 0x1002e8 + ((mr - 2) * 4), data);
-                hwsq_wr32(hwsq, 0x1002e0 + ((mr - 2) * 4), data);
-        }
-}
-
-static void
-mclk_clock_set(struct nouveau_mem_exec_func *exec)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        struct nv50_pm_state *info = exec->priv;
-        struct hwsq_ucode *hwsq = &info->mclk_hwsq;
-        u32 ctrl = nv_rd32(device, 0x004008);
-
-        info->mmast = nv_rd32(device, 0x00c040);
-        info->mmast &= ~0xc0000000; /* get MCLK_2 from HREF */
-        info->mmast |=  0x0000c000; /* use MCLK_2 as MPLL_BYPASS clock */
-
-        hwsq_wr32(hwsq, 0xc040, info->mmast);
-        hwsq_wr32(hwsq, 0x4008, ctrl | 0x00000200); /* bypass MPLL */
-        if (info->mctrl & 0x80000000)
-                hwsq_wr32(hwsq, 0x400c, info->mcoef);
-        hwsq_wr32(hwsq, 0x4008, info->mctrl);
-}
-
-static void
-mclk_timing_set(struct nouveau_mem_exec_func *exec)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        struct nv50_pm_state *info = exec->priv;
-        struct nouveau_pm_level *perflvl = info->perflvl;
-        struct hwsq_ucode *hwsq = &info->mclk_hwsq;
-        int i;
-
-        for (i = 0; i < 9; i++) {
-                u32 reg = 0x100220 + (i * 4);
-                u32 val = nv_rd32(device, reg);
-                if (val != perflvl->timing.reg[i])
-                        hwsq_wr32(hwsq, reg, perflvl->timing.reg[i]);
-        }
-}
-
-static int
-calc_mclk(struct drm_device *dev, struct nouveau_pm_level *perflvl,
-          struct nv50_pm_state *info)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nouveau_dev(dev);
-        u32 crtc_mask = 0; /*XXX: nv50_display_active_crtcs(dev); */
-        struct nouveau_mem_exec_func exec = {
-                .dev = dev,
-                .precharge = mclk_precharge,
-                .refresh = mclk_refresh,
-                .refresh_auto = mclk_refresh_auto,
-                .refresh_self = mclk_refresh_self,
-                .wait = mclk_wait,
-                .mrg = mclk_mrg,
-                .mrs = mclk_mrs,
-                .clock_set = mclk_clock_set,
-                .timing_set = mclk_timing_set,
-                .priv = info
-        };
-        struct hwsq_ucode *hwsq = &info->mclk_hwsq;
-        struct nvbios_pll pll;
-        int N, M, P;
-        int ret;
-
-        /* use pcie refclock if possible, otherwise use mpll */
-        info->mctrl  = nv_rd32(device, 0x004008);
-        info->mctrl &= ~0x81ff0200;
-        if (clk_same(perflvl->memory, read_clk(dev, clk_src_href))) {
-                info->mctrl |= 0x00000200 | (pll.bias_p << 19);
-        } else {
-                ret = calc_pll(dev, 0x4008, &pll, perflvl->memory, &N, &M, &P);
-                if (ret == 0)
-                        return -EINVAL;
-
-                info->mctrl |= 0x80000000 | (P << 22) | (P << 16);
-                info->mctrl |= pll.bias_p << 19;
-                info->mcoef  = (N << 8) | M;
-        }
-
-        /* build the ucode which will reclock the memory for us */
-        hwsq_init(hwsq);
-        if (crtc_mask) {
-                hwsq_op5f(hwsq, crtc_mask, 0x00); /* wait for scanout */
-                hwsq_op5f(hwsq, crtc_mask, 0x01); /* wait for vblank */
-        }
-        if (nv_device(drm->device)->chipset >= 0x92)
-                hwsq_wr32(hwsq, 0x611200, 0x00003300); /* disable scanout */
-        hwsq_setf(hwsq, 0x10, 0); /* disable bus access */
-        hwsq_op5f(hwsq, 0x00, 0x01); /* no idea :s */
-
-        ret = nouveau_mem_exec(&exec, perflvl);
-        if (ret)
-                return ret;
-
-        hwsq_setf(hwsq, 0x10, 1); /* enable bus access */
-        hwsq_op5f(hwsq, 0x00, 0x00); /* no idea, reverse of 0x00, 0x01? */
-        if (nv_device(drm->device)->chipset >= 0x92)
-                hwsq_wr32(hwsq, 0x611200, 0x00003330); /* enable scanout */
-        hwsq_fini(hwsq);
-        return 0;
-}
-
-void *
-nv50_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nv50_pm_state *info;
-        struct hwsq_ucode *hwsq;
-        struct nvbios_pll pll;
-        u32 out, mast, divs, ctrl;
-        int clk, ret = -EINVAL;
-        int N, M, P1, P2;
-
-        if (nv_device(drm->device)->chipset == 0xaa ||
-            nv_device(drm->device)->chipset == 0xac)
-                return ERR_PTR(-ENODEV);
-
-        info = kmalloc(sizeof(*info), GFP_KERNEL);
-        if (!info)
-                return ERR_PTR(-ENOMEM);
-        info->perflvl = perflvl;
-
-        /* memory: build hwsq ucode which we'll use to reclock memory.
-         *         use pcie refclock if possible, otherwise use mpll */
-        info->mclk_hwsq.len = 0;
-        if (perflvl->memory) {
-                ret = calc_mclk(dev, perflvl, info);
-                if (ret)
-                        goto error;
-                info->mscript = perflvl->memscript;
-        }
-
-        divs = read_div(dev);
-        mast = info->mmast;
-
-        /* start building HWSQ script for engine reclocking */
-        hwsq = &info->eclk_hwsq;
-        hwsq_init(hwsq);
-        hwsq_setf(hwsq, 0x10, 0); /* disable bus access */
-        hwsq_op5f(hwsq, 0x00, 0x01); /* wait for access disabled? */
-
-        /* vdec/dom6: switch to "safe" clocks temporarily */
-        if (perflvl->vdec) {
-                mast &= ~0x00000c00;
-                divs &= ~0x00000700;
-        }
-
-        if (perflvl->dom6) {
-                mast &= ~0x0c000000;
-                divs &= ~0x00000007;
-        }
-
-        hwsq_wr32(hwsq, 0x00c040, mast);
-
-        /* vdec: avoid modifying xpll until we know exactly how the other
-         * clock domains work, i suspect at least some of them can also be
-         * tied to xpll...
-         */
-        if (perflvl->vdec) {
-                /* see how close we can get using nvclk as a source */
-                clk = calc_div(perflvl->core, perflvl->vdec, &P1);
-
-                /* see how close we can get using xpll/hclk as a source */
-                if (nv_device(drm->device)->chipset != 0x98)
-                        out = read_pll(dev, 0x004030);
-                else
-                        out = read_clk(dev, clk_src_hclkm3d2);
-                out = calc_div(out, perflvl->vdec, &P2);
-
-                /* select whichever gets us closest */
-                if (abs((int)perflvl->vdec - clk) <=
-                    abs((int)perflvl->vdec - out)) {
-                        if (nv_device(drm->device)->chipset != 0x98)
-                                mast |= 0x00000c00;
-                        divs |= P1 << 8;
-                } else {
-                        mast |= 0x00000800;
-                        divs |= P2 << 8;
-                }
-        }
-
-        /* dom6: nfi what this is, but we're limited to various combinations
-         * of the host clock frequency
-         */
-        if (perflvl->dom6) {
-                if (clk_same(perflvl->dom6, read_clk(dev, clk_src_href))) {
-                        mast |= 0x00000000;
-                } else
-                if (clk_same(perflvl->dom6, read_clk(dev, clk_src_hclk))) {
-                        mast |= 0x08000000;
-                } else {
-                        clk = read_clk(dev, clk_src_hclk) * 3;
-                        clk = calc_div(clk, perflvl->dom6, &P1);
-
-                        mast |= 0x0c000000;
-                        divs |= P1;
-                }
-        }
-
-        /* vdec/dom6: complete switch to new clocks */
-        switch (nv_device(drm->device)->chipset) {
-        case 0x92:
-        case 0x94:
-        case 0x96:
-                hwsq_wr32(hwsq, 0x004800, divs);
-                break;
-        default:
-                hwsq_wr32(hwsq, 0x004700, divs);
-                break;
-        }
-
-        hwsq_wr32(hwsq, 0x00c040, mast);
-
-        /* core/shader: make sure sclk/nvclk are disconnected from their
-         * PLLs (nvclk to dom6, sclk to hclk)
-         */
-        if (nv_device(drm->device)->chipset < 0x92)
-                mast = (mast & ~0x001000b0) | 0x00100080;
-        else
-                mast = (mast & ~0x000000b3) | 0x00000081;
-
-        hwsq_wr32(hwsq, 0x00c040, mast);
-
-        /* core: for the moment at least, always use nvpll */
-        clk = calc_pll(dev, 0x4028, &pll, perflvl->core, &N, &M, &P1);
-        if (clk == 0)
-                goto error;
-
-        ctrl  = nv_rd32(device, 0x004028) & ~0xc03f0100;
-        mast &= ~0x00100000;
-        mast |= 3;
-
-        hwsq_wr32(hwsq, 0x004028, 0x80000000 | (P1 << 19) | (P1 << 16) | ctrl);
-        hwsq_wr32(hwsq, 0x00402c, (N << 8) | M);
-
-        /* shader: tie to nvclk if possible, otherwise use spll.  have to be
-         * very careful that the shader clock is at least twice the core, or
-         * some chipsets will be very unhappy.  i expect most or all of these
-         * cases will be handled by tying to nvclk, but it's possible there's
-         * corners
-         */
-        ctrl = nv_rd32(device, 0x004020) & ~0xc03f0100;
-
-        if (P1-- && perflvl->shader == (perflvl->core << 1)) {
-                hwsq_wr32(hwsq, 0x004020, (P1 << 19) | (P1 << 16) | ctrl);
-                hwsq_wr32(hwsq, 0x00c040, 0x00000020 | mast);
-        } else {
-                clk = calc_pll(dev, 0x4020, &pll, perflvl->shader, &N, &M, &P1);
-                if (clk == 0)
-                        goto error;
-                ctrl |= 0x80000000;
-
-                hwsq_wr32(hwsq, 0x004020, (P1 << 19) | (P1 << 16) | ctrl);
-                hwsq_wr32(hwsq, 0x004024, (N << 8) | M);
-                hwsq_wr32(hwsq, 0x00c040, 0x00000030 | mast);
-        }
-
-        hwsq_setf(hwsq, 0x10, 1); /* enable bus access */
-        hwsq_op5f(hwsq, 0x00, 0x00); /* wait for access enabled? */
-        hwsq_fini(hwsq);
-
-        return info;
-error:
-        kfree(info);
-        return ERR_PTR(ret);
-}
-
-static int
-prog_hwsq(struct drm_device *dev, struct hwsq_ucode *hwsq)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        u32 hwsq_data, hwsq_kick;
-        int i;
-
-        if (nv_device(drm->device)->chipset < 0x94) {
-                hwsq_data = 0x001400;
-                hwsq_kick = 0x00000003;
-        } else {
-                hwsq_data = 0x080000;
-                hwsq_kick = 0x00000001;
-        }
-        /* upload hwsq ucode */
-        nv_mask(device, 0x001098, 0x00000008, 0x00000000);
-        nv_wr32(device, 0x001304, 0x00000000);
-        if (nv_device(drm->device)->chipset >= 0x92)
-                nv_wr32(device, 0x001318, 0x00000000);
-        for (i = 0; i < hwsq->len / 4; i++)
-                nv_wr32(device, hwsq_data + (i * 4), hwsq->ptr.u32[i]);
-        nv_mask(device, 0x001098, 0x00000018, 0x00000018);
-
-        /* launch, and wait for completion */
-        nv_wr32(device, 0x00130c, hwsq_kick);
-        if (!nv_wait(device, 0x001308, 0x00000100, 0x00000000)) {
-                NV_ERROR(drm, "hwsq ucode exec timed out\n");
-                NV_ERROR(drm, "0x001308: 0x%08x\n", nv_rd32(device, 0x001308));
-                for (i = 0; i < hwsq->len / 4; i++) {
-                        NV_ERROR(drm, "0x%06x: 0x%08x\n", 0x1400 + (i * 4),
-                                 nv_rd32(device, 0x001400 + (i * 4)));
-                }
-
-                return -EIO;
-        }
-
-        return 0;
-}
-
-int
-nv50_pm_clocks_set(struct drm_device *dev, void *data)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nv50_pm_state *info = data;
-        struct bit_entry M;
-        int ret = -EBUSY;
-
-        /* halt and idle execution engines */
-        nv_mask(device, 0x002504, 0x00000001, 0x00000001);
-        if (!nv_wait(device, 0x002504, 0x00000010, 0x00000010))
-                goto resume;
-        if (!nv_wait(device, 0x00251c, 0x0000003f, 0x0000003f))
-                goto resume;
-
-        /* program memory clock, if necessary - must come before engine clock
-         * reprogramming due to how we construct the hwsq scripts in pre()
-         */
-#define nouveau_bios_init_exec(a,b) nouveau_bios_run_init_table((a), (b), NULL, 0)
-        if (info->mclk_hwsq.len) {
-                /* execute some scripts that do ??? from the vbios.. */
-                if (!bit_table(dev, 'M', &M) && M.version == 1) {
-                        if (M.length >= 6)
-                                nouveau_bios_init_exec(dev, ROM16(M.data[5]));
-                        if (M.length >= 8)
-                                nouveau_bios_init_exec(dev, ROM16(M.data[7]));
-                        if (M.length >= 10)
-                                nouveau_bios_init_exec(dev, ROM16(M.data[9]));
-                        nouveau_bios_init_exec(dev, info->mscript);
-                }
-
-                ret = prog_hwsq(dev, &info->mclk_hwsq);
-                if (ret)
-                        goto resume;
-        }
-
-        /* program engine clocks */
-        ret = prog_hwsq(dev, &info->eclk_hwsq);
-
-resume:
-        nv_mask(device, 0x002504, 0x00000001, 0x00000000);
-        kfree(info);
-        return ret;
-}
diff --git a/drivers/gpu/drm/nouveau/nva3_pm.c b/drivers/gpu/drm/nouveau/nva3_pm.c
deleted file mode 100644
index 0d0ed597fea8..000000000000
--- a/drivers/gpu/drm/nouveau/nva3_pm.c
+++ /dev/null
@@ -1,624 +0,0 @@
-/*
- * Copyright 2010 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include <drm/drmP.h>
-#include "nouveau_drm.h"
-#include "nouveau_bios.h"
-#include "nouveau_pm.h"
-
-#include <subdev/bios/pll.h>
-#include <subdev/bios.h>
-#include <subdev/clock.h>
-#include <subdev/timer.h>
-#include <subdev/fb.h>
-
-static u32 read_clk(struct drm_device *, int, bool);
-static u32 read_pll(struct drm_device *, int, u32);
-
-static u32
-read_vco(struct drm_device *dev, int clk)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        u32 sctl = nv_rd32(device, 0x4120 + (clk * 4));
-        if ((sctl & 0x00000030) != 0x00000030)
-                return read_pll(dev, 0x41, 0x00e820);
-        return read_pll(dev, 0x42, 0x00e8a0);
-}
-
-static u32
-read_clk(struct drm_device *dev, int clk, bool ignore_en)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        u32 sctl, sdiv, sclk;
-
-        /* refclk for the 0xe8xx plls is a fixed frequency */
-        if (clk >= 0x40) {
-                if (nv_device(drm->device)->chipset == 0xaf) {
-                        /* no joke.. seriously.. sigh.. */
-                        return nv_rd32(device, 0x00471c) * 1000;
-                }
-
-                return device->crystal;
-        }
-
-        sctl = nv_rd32(device, 0x4120 + (clk * 4));
-        if (!ignore_en && !(sctl & 0x00000100))
-                return 0;
-
-        switch (sctl & 0x00003000) {
-        case 0x00000000:
-                return device->crystal;
-        case 0x00002000:
-                if (sctl & 0x00000040)
-                        return 108000;
-                return 100000;
-        case 0x00003000:
-                sclk = read_vco(dev, clk);
-                sdiv = ((sctl & 0x003f0000) >> 16) + 2;
-                return (sclk * 2) / sdiv;
-        default:
-                return 0;
-        }
-}
-
-static u32
-read_pll(struct drm_device *dev, int clk, u32 pll)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        u32 ctrl = nv_rd32(device, pll + 0);
-        u32 sclk = 0, P = 1, N = 1, M = 1;
-
-        if (!(ctrl & 0x00000008)) {
-                if (ctrl & 0x00000001) {
-                        u32 coef = nv_rd32(device, pll + 4);
-                        M = (coef & 0x000000ff) >> 0;
-                        N = (coef & 0x0000ff00) >> 8;
-                        P = (coef & 0x003f0000) >> 16;
-
-                        /* no post-divider on these.. */
-                        if ((pll & 0x00ff00) == 0x00e800)
-                                P = 1;
-
-                        sclk = read_clk(dev, 0x00 + clk, false);
-                }
-        } else {
-                sclk = read_clk(dev, 0x10 + clk, false);
-        }
-
-        if (M * P)
-                return sclk * N / (M * P);
-        return 0;
-}
-
-struct creg {
-        u32 clk;
-        u32 pll;
-};
-
-static int
-calc_clk(struct drm_device *dev, int clk, u32 pll, u32 khz, struct creg *reg)
-{
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_bios *bios = nouveau_bios(device);
-        struct nvbios_pll limits;
-        u32 oclk, sclk, sdiv;
-        int P, N, M, diff;
-        int ret;
-
-        reg->pll = 0;
-        reg->clk = 0;
-        if (!khz) {
-                NV_DEBUG(drm, "no clock for 0x%04x/0x%02x\n", pll, clk);
-                return 0;
-        }
-
-        switch (khz) {
-        case 27000:
-                reg->clk = 0x00000100;
-                return khz;
-        case 100000:
-                reg->clk = 0x00002100;
-                return khz;
-        case 108000:
-                reg->clk = 0x00002140;
-                return khz;
-        default:
-                sclk = read_vco(dev, clk);
-                sdiv = min((sclk * 2) / (khz - 2999), (u32)65);
-                /* if the clock has a PLL attached, and we can get a within
-                 * [-2, 3) MHz of a divider, we'll disable the PLL and use
-                 * the divider instead.
-                 *
-                 * divider can go as low as 2, limited here because NVIDIA
-                 * and the VBIOS on my NVA8 seem to prefer using the PLL
-                 * for 810MHz - is there a good reason?
-                 */
-                if (sdiv > 4) {
-                        oclk = (sclk * 2) / sdiv;
-                        diff = khz - oclk;
-                        if (!pll || (diff >= -2000 && diff < 3000)) {
-                                reg->clk = (((sdiv - 2) << 16) | 0x00003100);
-                                return oclk;
-                        }
-                }
-
-                if (!pll) {
-                        NV_ERROR(drm, "bad freq %02x: %d %d\n", clk, khz, sclk);
-                        return -ERANGE;
-                }
-
-                break;
-        }
-
-        ret = nvbios_pll_parse(bios, pll, &limits);
-        if (ret)
-                return ret;
-
-        limits.refclk = read_clk(dev, clk - 0x10, true);
-        if (!limits.refclk)
-                return -EINVAL;
-
-        ret = nva3_calc_pll(dev, &limits, khz, &N, NULL, &M, &P);
-        if (ret >= 0) {
-                reg->clk = nv_rd32(device, 0x4120 + (clk * 4));
-                reg->pll = (P << 16) | (N << 8) | M;
-        }
-
-        return ret;
-}
-
-static void
-prog_pll(struct drm_device *dev, int clk, u32 pll, struct creg *reg)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        const u32 src0 = 0x004120 + (clk * 4);
-        const u32 src1 = 0x004160 + (clk * 4);
-        const u32 ctrl = pll + 0;
-        const u32 coef = pll + 4;
-
-        if (!reg->clk && !reg->pll) {
-                NV_DEBUG(drm, "no clock for %02x\n", clk);
-                return;
-        }
-
-        if (reg->pll) {
-                nv_mask(device, src0, 0x00000101, 0x00000101);
-                nv_wr32(device, coef, reg->pll);
-                nv_mask(device, ctrl, 0x00000015, 0x00000015);
-                nv_mask(device, ctrl, 0x00000010, 0x00000000);
-                nv_wait(device, ctrl, 0x00020000, 0x00020000);
-                nv_mask(device, ctrl, 0x00000010, 0x00000010);
-                nv_mask(device, ctrl, 0x00000008, 0x00000000);
-                nv_mask(device, src1, 0x00000100, 0x00000000);
-                nv_mask(device, src1, 0x00000001, 0x00000000);
-        } else {
-                nv_mask(device, src1, 0x003f3141, 0x00000101 | reg->clk);
-                nv_mask(device, ctrl, 0x00000018, 0x00000018);
-                udelay(20);
-                nv_mask(device, ctrl, 0x00000001, 0x00000000);
-                nv_mask(device, src0, 0x00000100, 0x00000000);
-                nv_mask(device, src0, 0x00000001, 0x00000000);
-        }
-}
-
-static void
-prog_clk(struct drm_device *dev, int clk, struct creg *reg)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-
-        if (!reg->clk) {
-                NV_DEBUG(drm, "no clock for %02x\n", clk);
-                return;
-        }
-
-        nv_mask(device, 0x004120 + (clk * 4), 0x003f3141, 0x00000101 | reg->clk);
-}
-
-int
-nva3_pm_clocks_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
-{
-        perflvl->core   = read_pll(dev, 0x00, 0x4200);
-        perflvl->shader = read_pll(dev, 0x01, 0x4220);
-        perflvl->memory = read_pll(dev, 0x02, 0x4000);
-        perflvl->unka0  = read_clk(dev, 0x20, false);
-        perflvl->vdec   = read_clk(dev, 0x21, false);
-        perflvl->daemon = read_clk(dev, 0x25, false);
-        perflvl->copy   = perflvl->core;
-        return 0;
-}
-
-struct nva3_pm_state {
-        struct nouveau_pm_level *perflvl;
-
-        struct creg nclk;
-        struct creg sclk;
-        struct creg vdec;
-        struct creg unka0;
-
-        struct creg mclk;
-        u8 *rammap;
-        u8  rammap_ver;
-        u8  rammap_len;
-        u8 *ramcfg;
-        u8  ramcfg_len;
-        u32 r004018;
-        u32 r100760;
-};
-
-void *
-nva3_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
-{
-        struct nva3_pm_state *info;
-        u8 ramcfg_cnt;
-        int ret;
-
-        info = kzalloc(sizeof(*info), GFP_KERNEL);
-        if (!info)
-                return ERR_PTR(-ENOMEM);
-
-        ret = calc_clk(dev, 0x10, 0x4200, perflvl->core, &info->nclk);
-        if (ret < 0)
-                goto out;
-
-        ret = calc_clk(dev, 0x11, 0x4220, perflvl->shader, &info->sclk);
-        if (ret < 0)
-                goto out;
-
-        ret = calc_clk(dev, 0x12, 0x4000, perflvl->memory, &info->mclk);
-        if (ret < 0)
-                goto out;
-
-        ret = calc_clk(dev, 0x20, 0x0000, perflvl->unka0, &info->unka0);
-        if (ret < 0)
-                goto out;
-
-        ret = calc_clk(dev, 0x21, 0x0000, perflvl->vdec, &info->vdec);
-        if (ret < 0)
-                goto out;
-
-        info->rammap = nouveau_perf_rammap(dev, perflvl->memory,
-                                           &info->rammap_ver,
-                                           &info->rammap_len,
-                                           &ramcfg_cnt, &info->ramcfg_len);
-        if (info->rammap_ver != 0x10 || info->rammap_len < 5)
-                info->rammap = NULL;
-
-        info->ramcfg = nouveau_perf_ramcfg(dev, perflvl->memory,
-                                           &info->rammap_ver,
-                                           &info->ramcfg_len);
-        if (info->rammap_ver != 0x10)
-                info->ramcfg = NULL;
-
-        info->perflvl = perflvl;
-out:
-        if (ret < 0) {
-                kfree(info);
-                info = ERR_PTR(ret);
-        }
-        return info;
-}
-
-static bool
-nva3_pm_grcp_idle(void *data)
-{
-        struct drm_device *dev = data;
-        struct nouveau_device *device = nouveau_dev(dev);
-
-        if (!(nv_rd32(device, 0x400304) & 0x00000001))
-                return true;
-        if (nv_rd32(device, 0x400308) == 0x0050001c)
-                return true;
-        return false;
-}
-
-static void
-mclk_precharge(struct nouveau_mem_exec_func *exec)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        nv_wr32(device, 0x1002d4, 0x00000001);
-}
-
-static void
-mclk_refresh(struct nouveau_mem_exec_func *exec)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        nv_wr32(device, 0x1002d0, 0x00000001);
-}
-
-static void
-mclk_refresh_auto(struct nouveau_mem_exec_func *exec, bool enable)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        nv_wr32(device, 0x100210, enable ? 0x80000000 : 0x00000000);
-}
-
-static void
-mclk_refresh_self(struct nouveau_mem_exec_func *exec, bool enable)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        nv_wr32(device, 0x1002dc, enable ? 0x00000001 : 0x00000000);
-}
-
-static void
-mclk_wait(struct nouveau_mem_exec_func *exec, u32 nsec)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        volatile u32 post = nv_rd32(device, 0); (void)post;
-        udelay((nsec + 500) / 1000);
-}
-
-static u32
-mclk_mrg(struct nouveau_mem_exec_func *exec, int mr)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        if (mr <= 1)
-                return nv_rd32(device, 0x1002c0 + ((mr - 0) * 4));
-        if (mr <= 3)
-                return nv_rd32(device, 0x1002e0 + ((mr - 2) * 4));
-        return 0;
-}
-
-static void
-mclk_mrs(struct nouveau_mem_exec_func *exec, int mr, u32 data)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        struct nouveau_fb *pfb = nouveau_fb(device);
-        if (mr <= 1) {
-                if (pfb->ram->ranks > 1)
-                        nv_wr32(device, 0x1002c8 + ((mr - 0) * 4), data);
-                nv_wr32(device, 0x1002c0 + ((mr - 0) * 4), data);
-        } else
-        if (mr <= 3) {
-                if (pfb->ram->ranks > 1)
-                        nv_wr32(device, 0x1002e8 + ((mr - 2) * 4), data);
-                nv_wr32(device, 0x1002e0 + ((mr - 2) * 4), data);
-        }
-}
-
-static void
-mclk_clock_set(struct nouveau_mem_exec_func *exec)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        struct nva3_pm_state *info = exec->priv;
-        u32 ctrl;
-
-        ctrl = nv_rd32(device, 0x004000);
-        if (!(ctrl & 0x00000008) && info->mclk.pll) {
-                nv_wr32(device, 0x004000, (ctrl |=  0x00000008));
-                nv_mask(device, 0x1110e0, 0x00088000, 0x00088000);
-                nv_wr32(device, 0x004018, 0x00001000);
-                nv_wr32(device, 0x004000, (ctrl &= ~0x00000001));
-                nv_wr32(device, 0x004004, info->mclk.pll);
-                nv_wr32(device, 0x004000, (ctrl |=  0x00000001));
-                udelay(64);
-                nv_wr32(device, 0x004018, 0x00005000 | info->r004018);
-                udelay(20);
-        } else
-        if (!info->mclk.pll) {
-                nv_mask(device, 0x004168, 0x003f3040, info->mclk.clk);
-                nv_wr32(device, 0x004000, (ctrl |= 0x00000008));
-                nv_mask(device, 0x1110e0, 0x00088000, 0x00088000);
-                nv_wr32(device, 0x004018, 0x0000d000 | info->r004018);
-        }
-
-        if (info->rammap) {
-                if (info->ramcfg && (info->rammap[4] & 0x08)) {
-                        u32 unk5a0 = (ROM16(info->ramcfg[5]) << 8) |
-                                      info->ramcfg[5];
-                        u32 unk5a4 = ROM16(info->ramcfg[7]);
-                        u32 unk804 = (info->ramcfg[9] & 0xf0) << 16 |
-                                     (info->ramcfg[3] & 0x0f) << 16 |
-                                     (info->ramcfg[9] & 0x0f) |
-                                     0x80000000;
-                        nv_wr32(device, 0x1005a0, unk5a0);
-                        nv_wr32(device, 0x1005a4, unk5a4);
-                        nv_wr32(device, 0x10f804, unk804);
-                        nv_mask(device, 0x10053c, 0x00001000, 0x00000000);
-                } else {
-                        nv_mask(device, 0x10053c, 0x00001000, 0x00001000);
-                        nv_mask(device, 0x10f804, 0x80000000, 0x00000000);
-                        nv_mask(device, 0x100760, 0x22222222, info->r100760);
-                        nv_mask(device, 0x1007a0, 0x22222222, info->r100760);
-                        nv_mask(device, 0x1007e0, 0x22222222, info->r100760);
-                }
-        }
-
-        if (info->mclk.pll) {
-                nv_mask(device, 0x1110e0, 0x00088000, 0x00011000);
-                nv_wr32(device, 0x004000, (ctrl &= ~0x00000008));
-        }
-}
-
-static void
-mclk_timing_set(struct nouveau_mem_exec_func *exec)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        struct nva3_pm_state *info = exec->priv;
-        struct nouveau_pm_level *perflvl = info->perflvl;
-        int i;
-
-        for (i = 0; i < 9; i++)
-                nv_wr32(device, 0x100220 + (i * 4), perflvl->timing.reg[i]);
-
-        if (info->ramcfg) {
-                u32 data = (info->ramcfg[2] & 0x08) ? 0x00000000 : 0x00001000;
-                nv_mask(device, 0x100200, 0x00001000, data);
-        }
-
-        if (info->ramcfg) {
-                u32 unk714 = nv_rd32(device, 0x100714) & ~0xf0000010;
-                u32 unk718 = nv_rd32(device, 0x100718) & ~0x00000100;
-                u32 unk71c = nv_rd32(device, 0x10071c) & ~0x00000100;
-                if ( (info->ramcfg[2] & 0x20))
-                        unk714 |= 0xf0000000;
-                if (!(info->ramcfg[2] & 0x04))
-                        unk714 |= 0x00000010;
-                nv_wr32(device, 0x100714, unk714);
-
-                if (info->ramcfg[2] & 0x01)
-                        unk71c |= 0x00000100;
-                nv_wr32(device, 0x10071c, unk71c);
-
-                if (info->ramcfg[2] & 0x02)
-                        unk718 |= 0x00000100;
-                nv_wr32(device, 0x100718, unk718);
-
-                if (info->ramcfg[2] & 0x10)
-                        nv_wr32(device, 0x111100, 0x48000000); /*XXX*/
-        }
-}
-
-static void
-prog_mem(struct drm_device *dev, struct nva3_pm_state *info)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_mem_exec_func exec = {
-                .dev = dev,
-                .precharge = mclk_precharge,
-                .refresh = mclk_refresh,
-                .refresh_auto = mclk_refresh_auto,
-                .refresh_self = mclk_refresh_self,
-                .wait = mclk_wait,
-                .mrg = mclk_mrg,
-                .mrs = mclk_mrs,
-                .clock_set = mclk_clock_set,
-                .timing_set = mclk_timing_set,
-                .priv = info
-        };
-        u32 ctrl;
-
-        /* XXX: where the fuck does 750MHz come from? */
-        if (info->perflvl->memory <= 750000) {
-                info->r004018 = 0x10000000;
-                info->r100760 = 0x22222222;
-        }
-
-        ctrl = nv_rd32(device, 0x004000);
-        if (ctrl & 0x00000008) {
-                if (info->mclk.pll) {
-                        nv_mask(device, 0x004128, 0x00000101, 0x00000101);
-                        nv_wr32(device, 0x004004, info->mclk.pll);
-                        nv_wr32(device, 0x004000, (ctrl |= 0x00000001));
-                        nv_wr32(device, 0x004000, (ctrl &= 0xffffffef));
-                        nv_wait(device, 0x004000, 0x00020000, 0x00020000);
-                        nv_wr32(device, 0x004000, (ctrl |= 0x00000010));
-                        nv_wr32(device, 0x004018, 0x00005000 | info->r004018);
-                        nv_wr32(device, 0x004000, (ctrl |= 0x00000004));
-                }
-        } else {
-                u32 ssel = 0x00000101;
-                if (info->mclk.clk)
-                        ssel |= info->mclk.clk;
-                else
-                        ssel |= 0x00080000; /* 324MHz, shouldn't matter... */
-                nv_mask(device, 0x004168, 0x003f3141, ctrl);
-        }
-
-        if (info->ramcfg) {
-                if (info->ramcfg[2] & 0x10) {
-                        nv_mask(device, 0x111104, 0x00000600, 0x00000000);
-                } else {
-                        nv_mask(device, 0x111100, 0x40000000, 0x40000000);
-                        nv_mask(device, 0x111104, 0x00000180, 0x00000000);
-                }
-        }
-        if (info->rammap && !(info->rammap[4] & 0x02))
-                nv_mask(device, 0x100200, 0x00000800, 0x00000000);
-        nv_wr32(device, 0x611200, 0x00003300);
-        if (!(info->ramcfg[2] & 0x10))
-                nv_wr32(device, 0x111100, 0x4c020000); /*XXX*/
-
-        nouveau_mem_exec(&exec, info->perflvl);
-
-        nv_wr32(device, 0x611200, 0x00003330);
-        if (info->rammap && (info->rammap[4] & 0x02))
-                nv_mask(device, 0x100200, 0x00000800, 0x00000800);
-        if (info->ramcfg) {
-                if (info->ramcfg[2] & 0x10) {
-                        nv_mask(device, 0x111104, 0x00000180, 0x00000180);
-                        nv_mask(device, 0x111100, 0x40000000, 0x00000000);
-                } else {
-                        nv_mask(device, 0x111104, 0x00000600, 0x00000600);
-                }
-        }
-
-        if (info->mclk.pll) {
-                nv_mask(device, 0x004168, 0x00000001, 0x00000000);
-                nv_mask(device, 0x004168, 0x00000100, 0x00000000);
-        } else {
-                nv_mask(device, 0x004000, 0x00000001, 0x00000000);
-                nv_mask(device, 0x004128, 0x00000001, 0x00000000);
-                nv_mask(device, 0x004128, 0x00000100, 0x00000000);
-        }
-}
-
-int
-nva3_pm_clocks_set(struct drm_device *dev, void *pre_state)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nva3_pm_state *info = pre_state;
-        int ret = -EAGAIN;
-
-        /* prevent any new grctx switches from starting */
-        nv_wr32(device, 0x400324, 0x00000000);
-        nv_wr32(device, 0x400328, 0x0050001c); /* wait flag 0x1c */
-        /* wait for any pending grctx switches to complete */
-        if (!nv_wait_cb(device, nva3_pm_grcp_idle, dev)) {
-                NV_ERROR(drm, "pm: ctxprog didn't go idle\n");
-                goto cleanup;
-        }
-        /* freeze PFIFO */
-        nv_mask(device, 0x002504, 0x00000001, 0x00000001);
-        if (!nv_wait(device, 0x002504, 0x00000010, 0x00000010)) {
-                NV_ERROR(drm, "pm: fifo didn't go idle\n");
-                goto cleanup;
-        }
-
-        prog_pll(dev, 0x00, 0x004200, &info->nclk);
-        prog_pll(dev, 0x01, 0x004220, &info->sclk);
-        prog_clk(dev, 0x20, &info->unka0);
-        prog_clk(dev, 0x21, &info->vdec);
-
-        if (info->mclk.clk || info->mclk.pll)
-                prog_mem(dev, info);
-
-        ret = 0;
-
-cleanup:
-        /* unfreeze PFIFO */
-        nv_mask(device, 0x002504, 0x00000001, 0x00000000);
-        /* restore ctxprog to normal */
-        nv_wr32(device, 0x400324, 0x00000000);
-        nv_wr32(device, 0x400328, 0x0070009c); /* set flag 0x1c */
-        /* unblock it if necessary */
-        if (nv_rd32(device, 0x400308) == 0x0050001c)
-                nv_mask(device, 0x400824, 0x10000000, 0x10000000);
-        kfree(info);
-        return ret;
-}
diff --git a/drivers/gpu/drm/nouveau/nvc0_pm.c b/drivers/gpu/drm/nouveau/nvc0_pm.c
deleted file mode 100644
index 3b7041cb013f..000000000000
--- a/drivers/gpu/drm/nouveau/nvc0_pm.c
+++ /dev/null
@@ -1,599 +0,0 @@
-/*
- * Copyright 2011 Red Hat Inc.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the "Software"),
- * to deal in the Software without restriction, including without limitation
- * the rights to use, copy, modify, merge, publish, distribute, sublicense,
- * and/or sell copies of the Software, and to permit persons to whom the
- * Software is furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
- * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
- * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
- * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
- * OTHER DEALINGS IN THE SOFTWARE.
- *
- * Authors: Ben Skeggs
- */
-
-#include "nouveau_drm.h"
-#include "nouveau_bios.h"
-#include "nouveau_pm.h"
-
-#include <subdev/bios/pll.h>
-#include <subdev/bios.h>
-#include <subdev/clock.h>
-#include <subdev/timer.h>
-#include <subdev/fb.h>
-
-static u32 read_div(struct drm_device *, int, u32, u32);
-static u32 read_pll(struct drm_device *, u32);
-
-static u32
-read_vco(struct drm_device *dev, u32 dsrc)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        u32 ssrc = nv_rd32(device, dsrc);
-        if (!(ssrc & 0x00000100))
-                return read_pll(dev, 0x00e800);
-        return read_pll(dev, 0x00e820);
-}
-
-static u32
-read_pll(struct drm_device *dev, u32 pll)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        u32 ctrl = nv_rd32(device, pll + 0);
-        u32 coef = nv_rd32(device, pll + 4);
-        u32 P = (coef & 0x003f0000) >> 16;
-        u32 N = (coef & 0x0000ff00) >> 8;
-        u32 M = (coef & 0x000000ff) >> 0;
-        u32 sclk, doff;
-
-        if (!(ctrl & 0x00000001))
-                return 0;
-
-        switch (pll & 0xfff000) {
-        case 0x00e000:
-                sclk = 27000;
-                P = 1;
-                break;
-        case 0x137000:
-                doff = (pll - 0x137000) / 0x20;
-                sclk = read_div(dev, doff, 0x137120, 0x137140);
-                break;
-        case 0x132000:
-                switch (pll) {
-                case 0x132000:
-                        sclk = read_pll(dev, 0x132020);
-                        break;
-                case 0x132020:
-                        sclk = read_div(dev, 0, 0x137320, 0x137330);
-                        break;
-                default:
-                        return 0;
-                }
-                break;
-        default:
-                return 0;
-        }
-
-        return sclk * N / M / P;
-}
-
-static u32
-read_div(struct drm_device *dev, int doff, u32 dsrc, u32 dctl)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        u32 ssrc = nv_rd32(device, dsrc + (doff * 4));
-        u32 sctl = nv_rd32(device, dctl + (doff * 4));
-
-        switch (ssrc & 0x00000003) {
-        case 0:
-                if ((ssrc & 0x00030000) != 0x00030000)
-                        return 27000;
-                return 108000;
-        case 2:
-                return 100000;
-        case 3:
-                if (sctl & 0x80000000) {
-                        u32 sclk = read_vco(dev, dsrc + (doff * 4));
-                        u32 sdiv = (sctl & 0x0000003f) + 2;
-                        return (sclk * 2) / sdiv;
-                }
-
-                return read_vco(dev, dsrc + (doff * 4));
-        default:
-                return 0;
-        }
-}
-
-static u32
-read_mem(struct drm_device *dev)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        u32 ssel = nv_rd32(device, 0x1373f0);
-        if (ssel & 0x00000001)
-                return read_div(dev, 0, 0x137300, 0x137310);
-        return read_pll(dev, 0x132000);
-}
-
-static u32
-read_clk(struct drm_device *dev, int clk)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        u32 sctl = nv_rd32(device, 0x137250 + (clk * 4));
-        u32 ssel = nv_rd32(device, 0x137100);
-        u32 sclk, sdiv;
-
-        if (ssel & (1 << clk)) {
-                if (clk < 7)
-                        sclk = read_pll(dev, 0x137000 + (clk * 0x20));
-                else
-                        sclk = read_pll(dev, 0x1370e0);
-                sdiv = ((sctl & 0x00003f00) >> 8) + 2;
-        } else {
-                sclk = read_div(dev, clk, 0x137160, 0x1371d0);
-                sdiv = ((sctl & 0x0000003f) >> 0) + 2;
-        }
-
-        if (sctl & 0x80000000)
-                return (sclk * 2) / sdiv;
-        return sclk;
-}
-
-int
-nvc0_pm_clocks_get(struct drm_device *dev, struct nouveau_pm_level *perflvl)
-{
-        perflvl->shader = read_clk(dev, 0x00);
-        perflvl->core   = perflvl->shader / 2;
-        perflvl->memory = read_mem(dev);
-        perflvl->rop    = read_clk(dev, 0x01);
-        perflvl->hub07  = read_clk(dev, 0x02);
-        perflvl->hub06  = read_clk(dev, 0x07);
-        perflvl->hub01  = read_clk(dev, 0x08);
-        perflvl->copy   = read_clk(dev, 0x09);
-        perflvl->daemon = read_clk(dev, 0x0c);
-        perflvl->vdec   = read_clk(dev, 0x0e);
-        return 0;
-}
-
-struct nvc0_pm_clock {
-        u32 freq;
-        u32 ssel;
-        u32 mdiv;
-        u32 dsrc;
-        u32 ddiv;
-        u32 coef;
-};
-
-struct nvc0_pm_state {
-        struct nouveau_pm_level *perflvl;
-        struct nvc0_pm_clock eng[16];
-        struct nvc0_pm_clock mem;
-};
-
-static u32
-calc_div(struct drm_device *dev, int clk, u32 ref, u32 freq, u32 *ddiv)
-{
-        u32 div = min((ref * 2) / freq, (u32)65);
-        if (div < 2)
-                div = 2;
-
-        *ddiv = div - 2;
-        return (ref * 2) / div;
-}
-
-static u32
-calc_src(struct drm_device *dev, int clk, u32 freq, u32 *dsrc, u32 *ddiv)
-{
-        u32 sclk;
-
-        /* use one of the fixed frequencies if possible */
-        *ddiv = 0x00000000;
-        switch (freq) {
-        case  27000:
-        case 108000:
-                *dsrc = 0x00000000;
-                if (freq == 108000)
-                        *dsrc |= 0x00030000;
-                return freq;
-        case 100000:
-                *dsrc = 0x00000002;
-                return freq;
-        default:
-                *dsrc = 0x00000003;
-                break;
-        }
-
-        /* otherwise, calculate the closest divider */
-        sclk = read_vco(dev, clk);
-        if (clk < 7)
-                sclk = calc_div(dev, clk, sclk, freq, ddiv);
-        return sclk;
-}
-
-static u32
-calc_pll(struct drm_device *dev, int clk, u32 freq, u32 *coef)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_bios *bios = nouveau_bios(device);
-        struct nvbios_pll limits;
-        int N, M, P, ret;
-
-        ret = nvbios_pll_parse(bios, 0x137000 + (clk * 0x20), &limits);
-        if (ret)
-                return 0;
-
-        limits.refclk = read_div(dev, clk, 0x137120, 0x137140);
-        if (!limits.refclk)
-                return 0;
-
-        ret = nva3_calc_pll(dev, &limits, freq, &N, NULL, &M, &P);
-        if (ret <= 0)
-                return 0;
-
-        *coef = (P << 16) | (N << 8) | M;
-        return ret;
-}
-
-/* A (likely rather simplified and incomplete) view of the clock tree
- *
- * Key:
- *
- * S: source select
- * D: divider
- * P: pll
- * F: switch
- *
- * Engine clocks:
- *
- * 137250(D) ---- 137100(F0) ---- 137160(S)/1371d0(D) ------------------- ref
- *                      (F1) ---- 1370X0(P) ---- 137120(S)/137140(D) ---- ref
- *
- * Not all registers exist for all clocks.  For example: clocks >= 8 don't
- * have their own PLL (all tied to clock 7's PLL when in PLL mode), nor do
- * they have the divider at 1371d0, though the source selection at 137160
- * still exists.  You must use the divider at 137250 for these instead.
- *
- * Memory clock:
- *
- * TBD, read_mem() above is likely very wrong...
- *
- */
-
-static int
-calc_clk(struct drm_device *dev, int clk, struct nvc0_pm_clock *info, u32 freq)
-{
-        u32 src0, div0, div1D, div1P = 0;
-        u32 clk0, clk1 = 0;
-
-        /* invalid clock domain */
-        if (!freq)
-                return 0;
-
-        /* first possible path, using only dividers */
-        clk0 = calc_src(dev, clk, freq, &src0, &div0);
-        clk0 = calc_div(dev, clk, clk0, freq, &div1D);
-
-        /* see if we can get any closer using PLLs */
-        if (clk0 != freq && (0x00004387 & (1 << clk))) {
-                if (clk < 7)
-                        clk1 = calc_pll(dev, clk, freq, &info->coef);
-                else
-                        clk1 = read_pll(dev, 0x1370e0);
-                clk1 = calc_div(dev, clk, clk1, freq, &div1P);
-        }
-
-        /* select the method which gets closest to target freq */
-        if (abs((int)freq - clk0) <= abs((int)freq - clk1)) {
-                info->dsrc = src0;
-                if (div0) {
-                        info->ddiv |= 0x80000000;
-                        info->ddiv |= div0 << 8;
-                        info->ddiv |= div0;
-                }
-                if (div1D) {
-                        info->mdiv |= 0x80000000;
-                        info->mdiv |= div1D;
-                }
-                info->ssel = 0;
-                info->freq = clk0;
-        } else {
-                if (div1P) {
-                        info->mdiv |= 0x80000000;
-                        info->mdiv |= div1P << 8;
-                }
-                info->ssel = (1 << clk);
-                info->freq = clk1;
-        }
-
-        return 0;
-}
-
-static int
-calc_mem(struct drm_device *dev, struct nvc0_pm_clock *info, u32 freq)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_bios *bios = nouveau_bios(device);
-        struct nvbios_pll pll;
-        int N, M, P, ret;
-        u32 ctrl;
-
-        /* mclk pll input freq comes from another pll, make sure it's on */
-        ctrl = nv_rd32(device, 0x132020);
-        if (!(ctrl & 0x00000001)) {
-                /* if not, program it to 567MHz.  nfi where this value comes
-                 * from - it looks like it's in the pll limits table for
-                 * 132000 but the binary driver ignores all my attempts to
-                 * change this value.
-                 */
-                nv_wr32(device, 0x137320, 0x00000103);
-                nv_wr32(device, 0x137330, 0x81200606);
-                nv_wait(device, 0x132020, 0x00010000, 0x00010000);
-                nv_wr32(device, 0x132024, 0x0001150f);
-                nv_mask(device, 0x132020, 0x00000001, 0x00000001);
-                nv_wait(device, 0x137390, 0x00020000, 0x00020000);
-                nv_mask(device, 0x132020, 0x00000004, 0x00000004);
-        }
-
-        /* for the moment, until the clock tree is better understood, use
-         * pll mode for all clock frequencies
-         */
-        ret = nvbios_pll_parse(bios, 0x132000, &pll);
-        if (ret == 0) {
-                pll.refclk = read_pll(dev, 0x132020);
-                if (pll.refclk) {
-                        ret = nva3_calc_pll(dev, &pll, freq, &N, NULL, &M, &P);
-                        if (ret > 0) {
-                                info->coef = (P << 16) | (N << 8) | M;
-                                return 0;
-                        }
-                }
-        }
-
-        return -EINVAL;
-}
-
-void *
-nvc0_pm_clocks_pre(struct drm_device *dev, struct nouveau_pm_level *perflvl)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nvc0_pm_state *info;
-        int ret;
-
-        info = kzalloc(sizeof(*info), GFP_KERNEL);
-        if (!info)
-                return ERR_PTR(-ENOMEM);
-
-        /* NFI why this is still in the performance table, the ROPCs appear
-         * to get their clock from clock 2 ("hub07", actually hub05 on this
-         * chip, but, anyway...) as well.  nvatiming confirms hub05 and ROP
-         * are always the same freq with the binary driver even when the
-         * performance table says they should differ.
-         */
-        if (device->chipset == 0xd9)
-                perflvl->rop = 0;
-
-        if ((ret = calc_clk(dev, 0x00, &info->eng[0x00], perflvl->shader)) ||
-            (ret = calc_clk(dev, 0x01, &info->eng[0x01], perflvl->rop)) ||
-            (ret = calc_clk(dev, 0x02, &info->eng[0x02], perflvl->hub07)) ||
-            (ret = calc_clk(dev, 0x07, &info->eng[0x07], perflvl->hub06)) ||
-            (ret = calc_clk(dev, 0x08, &info->eng[0x08], perflvl->hub01)) ||
-            (ret = calc_clk(dev, 0x09, &info->eng[0x09], perflvl->copy)) ||
-            (ret = calc_clk(dev, 0x0c, &info->eng[0x0c], perflvl->daemon)) ||
-            (ret = calc_clk(dev, 0x0e, &info->eng[0x0e], perflvl->vdec))) {
-                kfree(info);
-                return ERR_PTR(ret);
-        }
-
-        if (perflvl->memory) {
-                ret = calc_mem(dev, &info->mem, perflvl->memory);
-                if (ret) {
-                        kfree(info);
-                        return ERR_PTR(ret);
-                }
-        }
-
-        info->perflvl = perflvl;
-        return info;
-}
-
-static void
-prog_clk(struct drm_device *dev, int clk, struct nvc0_pm_clock *info)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-
-        /* program dividers at 137160/1371d0 first */
-        if (clk < 7 && !info->ssel) {
-                nv_mask(device, 0x1371d0 + (clk * 0x04), 0x80003f3f, info->ddiv);
-                nv_wr32(device, 0x137160 + (clk * 0x04), info->dsrc);
-        }
-
-        /* switch clock to non-pll mode */
-        nv_mask(device, 0x137100, (1 << clk), 0x00000000);
-        nv_wait(device, 0x137100, (1 << clk), 0x00000000);
-
-        /* reprogram pll */
-        if (clk < 7) {
-                /* make sure it's disabled first... */
-                u32 base = 0x137000 + (clk * 0x20);
-                u32 ctrl = nv_rd32(device, base + 0x00);
-                if (ctrl & 0x00000001) {
-                        nv_mask(device, base + 0x00, 0x00000004, 0x00000000);
-                        nv_mask(device, base + 0x00, 0x00000001, 0x00000000);
-                }
-                /* program it to new values, if necessary */
-                if (info->ssel) {
-                        nv_wr32(device, base + 0x04, info->coef);
-                        nv_mask(device, base + 0x00, 0x00000001, 0x00000001);
-                        nv_wait(device, base + 0x00, 0x00020000, 0x00020000);
-                        nv_mask(device, base + 0x00, 0x00020004, 0x00000004);
-                }
-        }
-
-        /* select pll/non-pll mode, and program final clock divider */
-        nv_mask(device, 0x137100, (1 << clk), info->ssel);
-        nv_wait(device, 0x137100, (1 << clk), info->ssel);
-        nv_mask(device, 0x137250 + (clk * 0x04), 0x00003f3f, info->mdiv);
-}
-
-static void
-mclk_precharge(struct nouveau_mem_exec_func *exec)
-{
-}
-
-static void
-mclk_refresh(struct nouveau_mem_exec_func *exec)
-{
-}
-
-static void
-mclk_refresh_auto(struct nouveau_mem_exec_func *exec, bool enable)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        nv_wr32(device, 0x10f210, enable ? 0x80000000 : 0x00000000);
-}
-
-static void
-mclk_refresh_self(struct nouveau_mem_exec_func *exec, bool enable)
-{
-}
-
-static void
-mclk_wait(struct nouveau_mem_exec_func *exec, u32 nsec)
-{
-        udelay((nsec + 500) / 1000);
-}
-
-static u32
-mclk_mrg(struct nouveau_mem_exec_func *exec, int mr)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        struct nouveau_fb *pfb = nouveau_fb(device);
-        if (pfb->ram->type != NV_MEM_TYPE_GDDR5) {
-                if (mr <= 1)
-                        return nv_rd32(device, 0x10f300 + ((mr - 0) * 4));
-                return nv_rd32(device, 0x10f320 + ((mr - 2) * 4));
-        } else {
-                if (mr == 0)
-                        return nv_rd32(device, 0x10f300 + (mr * 4));
-                else
-                if (mr <= 7)
-                        return nv_rd32(device, 0x10f32c + (mr * 4));
-                return nv_rd32(device, 0x10f34c);
-        }
-}
-
-static void
-mclk_mrs(struct nouveau_mem_exec_func *exec, int mr, u32 data)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        struct nouveau_fb *pfb = nouveau_fb(device);
-        if (pfb->ram->type != NV_MEM_TYPE_GDDR5) {
-                if (mr <= 1) {
-                        nv_wr32(device, 0x10f300 + ((mr - 0) * 4), data);
-                        if (pfb->ram->ranks > 1)
-                                nv_wr32(device, 0x10f308 + ((mr - 0) * 4), data);
-                } else
-                if (mr <= 3) {
-                        nv_wr32(device, 0x10f320 + ((mr - 2) * 4), data);
-                        if (pfb->ram->ranks > 1)
-                                nv_wr32(device, 0x10f328 + ((mr - 2) * 4), data);
-                }
-        } else {
-                if      (mr ==  0) nv_wr32(device, 0x10f300 + (mr * 4), data);
-                else if (mr <=  7) nv_wr32(device, 0x10f32c + (mr * 4), data);
-                else if (mr == 15) nv_wr32(device, 0x10f34c, data);
-        }
-}
-
-static void
-mclk_clock_set(struct nouveau_mem_exec_func *exec)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        struct nvc0_pm_state *info = exec->priv;
-        u32 ctrl = nv_rd32(device, 0x132000);
-
-        nv_wr32(device, 0x137360, 0x00000001);
-        nv_wr32(device, 0x137370, 0x00000000);
-        nv_wr32(device, 0x137380, 0x00000000);
-        if (ctrl & 0x00000001)
-                nv_wr32(device, 0x132000, (ctrl &= ~0x00000001));
-
-        nv_wr32(device, 0x132004, info->mem.coef);
-        nv_wr32(device, 0x132000, (ctrl |= 0x00000001));
-        nv_wait(device, 0x137390, 0x00000002, 0x00000002);
-        nv_wr32(device, 0x132018, 0x00005000);
-
-        nv_wr32(device, 0x137370, 0x00000001);
-        nv_wr32(device, 0x137380, 0x00000001);
-        nv_wr32(device, 0x137360, 0x00000000);
-}
-
-static void
-mclk_timing_set(struct nouveau_mem_exec_func *exec)
-{
-        struct nouveau_device *device = nouveau_dev(exec->dev);
-        struct nvc0_pm_state *info = exec->priv;
-        struct nouveau_pm_level *perflvl = info->perflvl;
-        int i;
-
-        for (i = 0; i < 5; i++)
-                nv_wr32(device, 0x10f290 + (i * 4), perflvl->timing.reg[i]);
-}
-
-static void
-prog_mem(struct drm_device *dev, struct nvc0_pm_state *info)
-{
-        struct nouveau_device *device = nouveau_dev(dev);
-        struct nouveau_mem_exec_func exec = {
-                .dev = dev,
-                .precharge = mclk_precharge,
-                .refresh = mclk_refresh,
-                .refresh_auto = mclk_refresh_auto,
-                .refresh_self = mclk_refresh_self,
-                .wait = mclk_wait,
-                .mrg = mclk_mrg,
-                .mrs = mclk_mrs,
-                .clock_set = mclk_clock_set,
-                .timing_set = mclk_timing_set,
-                .priv = info
-        };
-
-        if (device->chipset < 0xd0)
-                nv_wr32(device, 0x611200, 0x00003300);
-        else
-                nv_wr32(device, 0x62c000, 0x03030000);
-
-        nouveau_mem_exec(&exec, info->perflvl);
-
-        if (device->chipset < 0xd0)
-                nv_wr32(device, 0x611200, 0x00003330);
-        else
-                nv_wr32(device, 0x62c000, 0x03030300);
-}
-int
-nvc0_pm_clocks_set(struct drm_device *dev, void *data)
-{
-        struct nvc0_pm_state *info = data;
-        int i;
-
-        if (info->mem.coef)
-                prog_mem(dev, info);
-
-        for (i = 0; i < 16; i++) {
-                if (!info->eng[i].freq)
-                        continue;
-                prog_clk(dev, i, &info->eng[i]);
-        }
-
-        kfree(info);
-        return 0;
-}
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index 835caba026d3..5e827c29d194 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -107,10 +107,17 @@ void qxl_display_read_client_monitors_config(struct qxl_device *qdev)
                 qxl_io_log(qdev, "failed crc check for client_monitors_config,"
                                  " retrying\n");
         }
-        drm_helper_hpd_irq_event(qdev->ddev);
+
+        if (!drm_helper_hpd_irq_event(qdev->ddev)) {
+                /* notify that the monitor configuration changed, to
+                   adjust at the arbitrary resolution */
+                drm_kms_helper_hotplug_event(qdev->ddev);
+        }
 }
 
-static int qxl_add_monitors_config_modes(struct drm_connector *connector)
+static int qxl_add_monitors_config_modes(struct drm_connector *connector,
+                                         unsigned *pwidth,
+                                         unsigned *pheight)
 {
         struct drm_device *dev = connector->dev;
         struct qxl_device *qdev = dev->dev_private;
@@ -126,11 +133,15 @@ static int qxl_add_monitors_config_modes(struct drm_connector *connector)
         mode = drm_cvt_mode(dev, head->width, head->height, 60, false, false,
                             false);
         mode->type |= DRM_MODE_TYPE_PREFERRED;
+        *pwidth = head->width;
+        *pheight = head->height;
         drm_mode_probed_add(connector, mode);
         return 1;
 }
 
-static int qxl_add_common_modes(struct drm_connector *connector)
+static int qxl_add_common_modes(struct drm_connector *connector,
+                                unsigned pwidth,
+                                unsigned pheight)
 {
         struct drm_device *dev = connector->dev;
         struct drm_display_mode *mode = NULL;
@@ -159,12 +170,9 @@ static int qxl_add_common_modes(struct drm_connector *connector)
         };
 
         for (i = 0; i < ARRAY_SIZE(common_modes); i++) {
-                if (common_modes[i].w < 320 || common_modes[i].h < 200)
-                        continue;
-
                 mode = drm_cvt_mode(dev, common_modes[i].w, common_modes[i].h,
                                     60, false, false, false);
-                if (common_modes[i].w == 1024 && common_modes[i].h == 768)
+                if (common_modes[i].w == pwidth && common_modes[i].h == pheight)
                         mode->type |= DRM_MODE_TYPE_PREFERRED;
                 drm_mode_probed_add(connector, mode);
         }
@@ -720,16 +728,18 @@ static int qxl_conn_get_modes(struct drm_connector *connector)
 {
         int ret = 0;
         struct qxl_device *qdev = connector->dev->dev_private;
+        unsigned pwidth = 1024;
+        unsigned pheight = 768;
 
         DRM_DEBUG_KMS("monitors_config=%p\n", qdev->monitors_config);
         /* TODO: what should we do here? only show the configured modes for the
          * device, or allow the full list, or both? */
         if (qdev->monitors_config && qdev->monitors_config->count) {
-                ret = qxl_add_monitors_config_modes(connector);
+                ret = qxl_add_monitors_config_modes(connector, &pwidth, &pheight);
                 if (ret < 0)
                         return ret;
         }
-        ret += qxl_add_common_modes(connector);
+        ret += qxl_add_common_modes(connector, pwidth, pheight);
         return ret;
 }
 
@@ -793,7 +803,10 @@ static enum drm_connector_status qxl_conn_detect(
                      qdev->client_monitors_config->count > output->index &&
                      qxl_head_enabled(&qdev->client_monitors_config->heads[output->index]));
 
-        DRM_DEBUG("\n");
+        DRM_DEBUG("#%d connected: %d\n", output->index, connected);
+        if (!connected)
+                qxl_monitors_config_set(qdev, output->index, 0, 0, 0, 0, 0);
+
         return connected ? connector_status_connected
                          : connector_status_disconnected;
 }
@@ -835,8 +848,21 @@ static const struct drm_encoder_funcs qxl_enc_funcs = {
         .destroy = qxl_enc_destroy,
 };
 
+static int qxl_mode_create_hotplug_mode_update_property(struct qxl_device *qdev)
+{
+        if (qdev->hotplug_mode_update_property)
+                return 0;
+
+        qdev->hotplug_mode_update_property =
+                drm_property_create_range(qdev->ddev, DRM_MODE_PROP_IMMUTABLE,
+                                          "hotplug_mode_update", 0, 1);
+
+        return 0;
+}
+
 static int qdev_output_init(struct drm_device *dev, int num_output)
 {
+        struct qxl_device *qdev = dev->dev_private;
         struct qxl_output *qxl_output;
         struct drm_connector *connector;
         struct drm_encoder *encoder;
@@ -863,6 +889,8 @@ static int qdev_output_init(struct drm_device *dev, int num_output)
         drm_encoder_helper_add(encoder, &qxl_enc_helper_funcs);
         drm_connector_helper_add(connector, &qxl_connector_helper_funcs);
 
+        drm_object_attach_property(&connector->base,
+                                   qdev->hotplug_mode_update_property, 0);
         drm_sysfs_connector_add(connector);
         return 0;
 }
@@ -975,6 +1003,9 @@ int qxl_modeset_init(struct qxl_device *qdev)
         qdev->ddev->mode_config.max_height = 8192;
 
         qdev->ddev->mode_config.fb_base = qdev->vram_base;
+
+        qxl_mode_create_hotplug_mode_update_property(qdev);
+
         for (i = 0 ; i < qxl_num_crtc; ++i) {
                 qdev_crtc_init(qdev->ddev, i);
                 qdev_output_init(qdev->ddev, i);
diff --git a/drivers/gpu/drm/qxl/qxl_drv.h b/drivers/gpu/drm/qxl/qxl_drv.h
index 41d22ed26060..7bda32f68d3b 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.h
+++ b/drivers/gpu/drm/qxl/qxl_drv.h
@@ -323,6 +323,8 @@ struct qxl_device {
         struct work_struct gc_work;
 
         struct work_struct fb_work;
+
+        struct drm_property *hotplug_mode_update_property;
 };
 
 /* forward declaration for QXL_INFO_IO */
diff --git a/drivers/gpu/drm/qxl/qxl_fb.c b/drivers/gpu/drm/qxl/qxl_fb.c
index 88722f233430..f437b30ce689 100644
--- a/drivers/gpu/drm/qxl/qxl_fb.c
+++ b/drivers/gpu/drm/qxl/qxl_fb.c
@@ -108,7 +108,7 @@ static void qxl_fb_dirty_flush(struct fb_info *info)
         u32 x1, x2, y1, y2;
 
         /* TODO: hard coding 32 bpp */
-        int stride = qfbdev->qfb.base.pitches[0] * 4;
+        int stride = qfbdev->qfb.base.pitches[0];
 
         x1 = qfbdev->dirty.x1;
         x2 = qfbdev->dirty.x2;
diff --git a/drivers/gpu/drm/qxl/qxl_kms.c b/drivers/gpu/drm/qxl/qxl_kms.c
index 9e8da9ee9731..e5ca498be920 100644
--- a/drivers/gpu/drm/qxl/qxl_kms.c
+++ b/drivers/gpu/drm/qxl/qxl_kms.c
@@ -120,7 +120,7 @@ int qxl_device_init(struct qxl_device *qdev,
                     struct pci_dev *pdev,
                     unsigned long flags)
 {
-        int r;
+        int r, sb;
 
         qdev->dev = &pdev->dev;
         qdev->ddev = ddev;
@@ -136,21 +136,39 @@ int qxl_device_init(struct qxl_device *qdev,
         qdev->rom_base = pci_resource_start(pdev, 2);
         qdev->rom_size = pci_resource_len(pdev, 2);
         qdev->vram_base = pci_resource_start(pdev, 0);
-        qdev->surfaceram_base = pci_resource_start(pdev, 1);
-        qdev->surfaceram_size = pci_resource_len(pdev, 1);
         qdev->io_base = pci_resource_start(pdev, 3);
 
         qdev->vram_mapping = io_mapping_create_wc(qdev->vram_base, pci_resource_len(pdev, 0));
-        qdev->surface_mapping = io_mapping_create_wc(qdev->surfaceram_base, qdev->surfaceram_size);
-        DRM_DEBUG_KMS("qxl: vram %llx-%llx(%dM %dk), surface %llx-%llx(%dM %dk)\n",
+
+        if (pci_resource_len(pdev, 4) > 0) {
+                /* 64bit surface bar present */
+                sb = 4;
+                qdev->surfaceram_base = pci_resource_start(pdev, sb);
+                qdev->surfaceram_size = pci_resource_len(pdev, sb);
+                qdev->surface_mapping =
+                        io_mapping_create_wc(qdev->surfaceram_base,
+                                             qdev->surfaceram_size);
+        }
+        if (qdev->surface_mapping == NULL) {
+                /* 64bit surface bar not present (or mapping failed) */
+                sb = 1;
+                qdev->surfaceram_base = pci_resource_start(pdev, sb);
+                qdev->surfaceram_size = pci_resource_len(pdev, sb);
+                qdev->surface_mapping =
+                        io_mapping_create_wc(qdev->surfaceram_base,
+                                             qdev->surfaceram_size);
+        }
+
+        DRM_DEBUG_KMS("qxl: vram %llx-%llx(%dM %dk), surface %llx-%llx(%dM %dk, %s)\n",
                  (unsigned long long)qdev->vram_base,
                  (unsigned long long)pci_resource_end(pdev, 0),
                  (int)pci_resource_len(pdev, 0) / 1024 / 1024,
                  (int)pci_resource_len(pdev, 0) / 1024,
                  (unsigned long long)qdev->surfaceram_base,
-                 (unsigned long long)pci_resource_end(pdev, 1),
+                 (unsigned long long)pci_resource_end(pdev, sb),
                  (int)qdev->surfaceram_size / 1024 / 1024,
-                 (int)qdev->surfaceram_size / 1024);
+                 (int)qdev->surfaceram_size / 1024,
+                 (sb == 4) ? "64bit" : "32bit");
 
         qdev->rom = ioremap(qdev->rom_base, qdev->rom_size);
         if (!qdev->rom) {
@@ -230,9 +248,13 @@ int qxl_device_init(struct qxl_device *qdev,
         qdev->surfaces_mem_slot = setup_slot(qdev, 1,
                 (unsigned long)qdev->surfaceram_base,
                 (unsigned long)qdev->surfaceram_base + qdev->surfaceram_size);
-        DRM_INFO("main mem slot %d [%lx,%x)\n",
-                qdev->main_mem_slot,
-                (unsigned long)qdev->vram_base, qdev->rom->ram_header_offset);
+        DRM_INFO("main mem slot %d [%lx,%x]\n",
+                 qdev->main_mem_slot,
+                 (unsigned long)qdev->vram_base, qdev->rom->ram_header_offset);
+        DRM_INFO("surface mem slot %d [%lx,%lx]\n",
+                 qdev->surfaces_mem_slot,
+                 (unsigned long)qdev->surfaceram_base,
+                 (unsigned long)qdev->surfaceram_size);
 
 
         qdev->gc_queue = create_singlethread_workqueue("qxl_gc");
diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c
index 037786d7c1dc..c7e7e6590c2b 100644
--- a/drivers/gpu/drm/qxl/qxl_ttm.c
+++ b/drivers/gpu/drm/qxl/qxl_ttm.c
@@ -516,6 +516,8 @@ int qxl_ttm_init(struct qxl_device *qdev)
                  (unsigned)qdev->vram_size / (1024 * 1024));
         DRM_INFO("qxl: %luM of IO pages memory ready (VRAM domain)\n",
                  ((unsigned)num_io_pages * PAGE_SIZE) / (1024 * 1024));
+        DRM_INFO("qxl: %uM of Surface memory size\n",
+                 (unsigned)qdev->surfaceram_size / (1024 * 1024));
         if (unlikely(qdev->mman.bdev.dev_mapping == NULL))
                 qdev->mman.bdev.dev_mapping = qdev->ddev->dev_mapping;
         r = qxl_ttm_debugfs_init(qdev);
diff --git a/drivers/gpu/drm/radeon/atombios.h b/drivers/gpu/drm/radeon/atombios.h
index af10f8571d87..92be50c39ffd 100644
--- a/drivers/gpu/drm/radeon/atombios.h
+++ b/drivers/gpu/drm/radeon/atombios.h
@@ -1711,7 +1711,9 @@ typedef struct _PIXEL_CLOCK_PARAMETERS_V6
 #define PIXEL_CLOCK_V6_MISC_HDMI_BPP_MASK           0x0c
 #define PIXEL_CLOCK_V6_MISC_HDMI_24BPP              0x00
 #define PIXEL_CLOCK_V6_MISC_HDMI_36BPP              0x04
+#define PIXEL_CLOCK_V6_MISC_HDMI_36BPP_V6           0x08    //for V6, the correct defintion for 36bpp should be 2 for 36bpp(2:1)
 #define PIXEL_CLOCK_V6_MISC_HDMI_30BPP              0x08
+#define PIXEL_CLOCK_V6_MISC_HDMI_30BPP_V6           0x04    //for V6, the correct defintion for 30bpp should be 1 for 36bpp(5:4)
 #define PIXEL_CLOCK_V6_MISC_HDMI_48BPP              0x0c
 #define PIXEL_CLOCK_V6_MISC_REF_DIV_SRC             0x10
 #define PIXEL_CLOCK_V6_MISC_GEN_DPREFCLK            0x40
@@ -2223,7 +2225,7 @@ typedef struct	_SET_VOLTAGE_PARAMETERS_V2
   USHORT   usVoltageLevel;              // real voltage level
 }SET_VOLTAGE_PARAMETERS_V2;
 
-
+// used by both SetVoltageTable v1.3 and v1.4
 typedef struct  _SET_VOLTAGE_PARAMETERS_V1_3
 {
   UCHAR    ucVoltageType;               // To tell which voltage to set up, VDDC/MVDDC/MVDDQ/VDDCI
@@ -2290,15 +2292,36 @@ typedef struct  _GET_LEAKAGE_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_1
 #define ATOM_GET_VOLTAGE_VID                0x00
 #define ATOM_GET_VOTLAGE_INIT_SEQ           0x03
 #define ATOM_GET_VOLTTAGE_PHASE_PHASE_VID   0x04
-// for SI, this state map to 0xff02 voltage state in Power Play table, which is power boost state
-#define ATOM_GET_VOLTAGE_STATE0_LEAKAGE_VID 0x10
+#define ATOM_GET_VOLTAGE_SVID2              0x07        //Get SVI2 Regulator Info
 
+// for SI, this state map to 0xff02 voltage state in Power Play table, which is power boost state
+#define ATOM_GET_VOLTAGE_STATE0_LEAKAGE_VID 0x10
 // for SI, this state map to 0xff01 voltage state in Power Play table, which is performance state
 #define ATOM_GET_VOLTAGE_STATE1_LEAKAGE_VID 0x11
-// undefined power state
+
 #define ATOM_GET_VOLTAGE_STATE2_LEAKAGE_VID 0x12
 #define ATOM_GET_VOLTAGE_STATE3_LEAKAGE_VID 0x13
 
+// New Added from CI Hawaii for GetVoltageInfoTable, input parameter structure
+typedef struct  _GET_VOLTAGE_INFO_INPUT_PARAMETER_V1_2
+{
+  UCHAR    ucVoltageType;               // Input: To tell which voltage to set up, VDDC/MVDDC/MVDDQ/VDDCI
+  UCHAR    ucVoltageMode;               // Input: Indicate action: Get voltage info
+  USHORT   usVoltageLevel;              // Input: real voltage level in unit of mv or Voltage Phase (0, 1, 2, .. ) or Leakage Id 
+  ULONG    ulSCLKFreq;                  // Input: when ucVoltageMode= ATOM_GET_VOLTAGE_EVV_VOLTAGE, DPM state SCLK frequency, Define in PPTable SCLK/Voltage dependence table
+}GET_VOLTAGE_INFO_INPUT_PARAMETER_V1_2;
+
+// New in GetVoltageInfo v1.2 ucVoltageMode
+#define ATOM_GET_VOLTAGE_EVV_VOLTAGE        0x09        
+
+// New Added from CI Hawaii for EVV feature 
+typedef struct  _GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2
+{
+  USHORT   usVoltageLevel;                               // real voltage level in unit of mv
+  USHORT   usVoltageId;                                  // Voltage Id programmed in Voltage Regulator
+  ULONG    ulReseved;
+}GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2;
+
 /****************************************************************************/  
 // Structures used by TVEncoderControlTable
 /****************************************************************************/  
@@ -3864,6 +3887,8 @@ typedef struct _ATOM_GPIO_PIN_ASSIGNMENT
 #define PP_AC_DC_SWITCH_GPIO_PINID          60
 //from SMU7.x, if ucGPIO_ID=VDDC_REGULATOR_VRHOT_GPIO_PINID in GPIO_LUTable, VRHot feature is enable
 #define VDDC_VRHOT_GPIO_PINID               61
+//if ucGPIO_ID=VDDC_PCC_GPIO_PINID in GPIO_LUTable, Peak Current Control feature is enabled
+#define VDDC_PCC_GPIO_PINID                 62
 
 typedef struct _ATOM_GPIO_PIN_LUT
 {
@@ -4169,10 +4194,10 @@ typedef struct _ATOM_COMMON_RECORD_HEADER
 #define ATOM_OBJECT_LINK_RECORD_TYPE                   18 //Once this record is present under one object, it indicats the oobject is linked to another obj described by the record
 #define ATOM_CONNECTOR_REMOTE_CAP_RECORD_TYPE          19
 #define ATOM_ENCODER_CAP_RECORD_TYPE                   20
-
+#define ATOM_BRACKET_LAYOUT_RECORD_TYPE                21
 
 //Must be updated when new record type is added,equal to that record definition!
-#define ATOM_MAX_OBJECT_RECORD_NUMBER             ATOM_ENCODER_CAP_RECORD_TYPE
+#define ATOM_MAX_OBJECT_RECORD_NUMBER             ATOM_BRACKET_LAYOUT_RECORD_TYPE
 
 typedef struct  _ATOM_I2C_RECORD
 {
@@ -4397,6 +4422,31 @@ typedef struct _ATOM_CONNECTOR_REMOTE_CAP_RECORD
   USHORT                      usReserved;
 }ATOM_CONNECTOR_REMOTE_CAP_RECORD;
 
+typedef struct  _ATOM_CONNECTOR_LAYOUT_INFO
+{
+   USHORT usConnectorObjectId;
+   UCHAR  ucConnectorType;
+   UCHAR  ucPosition;
+}ATOM_CONNECTOR_LAYOUT_INFO;
+
+// define ATOM_CONNECTOR_LAYOUT_INFO.ucConnectorType to describe the display connector size
+#define CONNECTOR_TYPE_DVI_D                 1
+#define CONNECTOR_TYPE_DVI_I                 2
+#define CONNECTOR_TYPE_VGA                   3
+#define CONNECTOR_TYPE_HDMI                  4
+#define CONNECTOR_TYPE_DISPLAY_PORT          5
+#define CONNECTOR_TYPE_MINI_DISPLAY_PORT     6
+
+typedef struct  _ATOM_BRACKET_LAYOUT_RECORD
+{
+  ATOM_COMMON_RECORD_HEADER   sheader;
+  UCHAR                       ucLength;
+  UCHAR                       ucWidth;
+  UCHAR                       ucConnNum;
+  UCHAR                       ucReserved;
+  ATOM_CONNECTOR_LAYOUT_INFO  asConnInfo[1];
+}ATOM_BRACKET_LAYOUT_RECORD;
+
 /****************************************************************************/  
 // ASIC voltage data table
 /****************************************************************************/  
@@ -4524,8 +4574,9 @@ typedef struct _ATOM_VOLTAGE_OBJECT_HEADER_V3{
 #define VOLTAGE_OBJ_VR_I2C_INIT_SEQ          3        //VOLTAGE REGULATOR INIT sequece through I2C -> ATOM_I2C_VOLTAGE_OBJECT_V3
 #define VOLTAGE_OBJ_PHASE_LUT                4        //Set Vregulator Phase lookup table ->ATOM_GPIO_VOLTAGE_OBJECT_V3
 #define VOLTAGE_OBJ_SVID2                    7        //Indicate voltage control by SVID2 ->ATOM_SVID2_VOLTAGE_OBJECT_V3
-#define VOLTAGE_OBJ_PWRBOOST_LEAKAGE_LUT     0x10     //Powerboost Voltage and LeakageId lookup table->ATOM_LEAKAGE_VOLTAGE_OBJECT_V3
-#define VOLTAGE_OBJ_HIGH_STATE_LEAKAGE_LUT   0x11     //High voltage state Voltage and LeakageId lookup table->ATOM_LEAKAGE_VOLTAGE_OBJECT_V3
+#define VOLTAGE_OBJ_EVV                      8 
+#define VOLTAGE_OBJ_PWRBOOST_LEAKAGE_LUT     0x10     //Powerboost Voltage and LeakageId lookup table->ATOM_LEAKAGE_VOLTAGE_OBJECT_V3
+#define VOLTAGE_OBJ_HIGH_STATE_LEAKAGE_LUT   0x11     //High voltage state Voltage and LeakageId lookup table->ATOM_LEAKAGE_VOLTAGE_OBJECT_V3
 #define VOLTAGE_OBJ_HIGH1_STATE_LEAKAGE_LUT  0x12     //High1 voltage state Voltage and LeakageId lookup table->ATOM_LEAKAGE_VOLTAGE_OBJECT_V3
 
 typedef struct  _VOLTAGE_LUT_ENTRY_V2
@@ -4552,6 +4603,10 @@ typedef struct  _ATOM_I2C_VOLTAGE_OBJECT_V3
    VOLTAGE_LUT_ENTRY asVolI2cLut[1];        // end with 0xff
 }ATOM_I2C_VOLTAGE_OBJECT_V3;
 
+// ATOM_I2C_VOLTAGE_OBJECT_V3.ucVoltageControlFlag
+#define VOLTAGE_DATA_ONE_BYTE                0
+#define VOLTAGE_DATA_TWO_BYTE                1
+
 typedef struct  _ATOM_GPIO_VOLTAGE_OBJECT_V3
 {
    ATOM_VOLTAGE_OBJECT_HEADER_V3 sHeader;   // voltage mode = VOLTAGE_OBJ_GPIO_LUT or VOLTAGE_OBJ_PHASE_LUT
@@ -4584,7 +4639,8 @@ typedef struct  _ATOM_SVID2_VOLTAGE_OBJECT_V3
 // 1:0 – offset trim, 
    USHORT   usLoadLine_PSI;    
 // GPU GPIO pin Id to SVID2 regulator VRHot pin. possible value 0~31. 0 means GPIO0, 31 means GPIO31
-   UCHAR    ucReserved[2];
+   UCHAR    ucSVDGpioId;     //0~31 indicate GPIO0~31
+   UCHAR    ucSVCGpioId;     //0~31 indicate GPIO0~31
    ULONG    ulReserved;
 }ATOM_SVID2_VOLTAGE_OBJECT_V3;
 
@@ -4637,6 +4693,49 @@ typedef struct  _ATOM_ASIC_PROFILING_INFO_V2_1
   USHORT usElbVDDCI_LevelArrayOffset;    // offset of 2 dimension voltage level USHORT array
 }ATOM_ASIC_PROFILING_INFO_V2_1;
 
+typedef struct  _ATOM_ASIC_PROFILING_INFO_V3_1
+{
+  ATOM_COMMON_TABLE_HEADER         asHeader; 
+  ULONG  ulEvvDerateTdp;
+  ULONG  ulEvvDerateTdc;
+  ULONG  ulBoardCoreTemp;
+  ULONG  ulMaxVddc;
+  ULONG  ulMinVddc;
+  ULONG  ulLoadLineSlop;
+  ULONG  ulLeakageTemp;
+  ULONG  ulLeakageVoltage;
+  ULONG  ulCACmEncodeRange;
+  ULONG  ulCACmEncodeAverage;
+  ULONG  ulCACbEncodeRange;
+  ULONG  ulCACbEncodeAverage;
+  ULONG  ulKt_bEncodeRange;
+  ULONG  ulKt_bEncodeAverage;
+  ULONG  ulKv_mEncodeRange;
+  ULONG  ulKv_mEncodeAverage;
+  ULONG  ulKv_bEncodeRange;
+  ULONG  ulKv_bEncodeAverage;
+  ULONG  ulLkgEncodeLn_MaxDivMin;
+  ULONG  ulLkgEncodeMin;
+  ULONG  ulEfuseLogisticAlpha;
+  USHORT usPowerDpm0;
+  USHORT usCurrentDpm0;
+  USHORT usPowerDpm1;
+  USHORT usCurrentDpm1;
+  USHORT usPowerDpm2;
+  USHORT usCurrentDpm2;
+  USHORT usPowerDpm3;
+  USHORT usCurrentDpm3;
+  USHORT usPowerDpm4;
+  USHORT usCurrentDpm4;
+  USHORT usPowerDpm5;
+  USHORT usCurrentDpm5;
+  USHORT usPowerDpm6;
+  USHORT usCurrentDpm6;
+  USHORT usPowerDpm7;
+  USHORT usCurrentDpm7;
+}ATOM_ASIC_PROFILING_INFO_V3_1;
+
+
 typedef struct _ATOM_POWER_SOURCE_OBJECT
 {
         UCHAR   ucPwrSrcId;                                                                                                     // Power source
@@ -5808,6 +5907,8 @@ typedef struct _ATOM_ASIC_INTERNAL_SS_INFO_V3
 #define ATOM_S7_DOS_MODE_PIXEL_DEPTHb0      0x0C
 #define ATOM_S7_DOS_MODE_PIXEL_FORMATb0     0xF0
 #define ATOM_S7_DOS_8BIT_DAC_ENb1           0x01
+#define ATOM_S7_ASIC_INIT_COMPLETEb1        0x02
+#define ATOM_S7_ASIC_INIT_COMPLETE_MASK     0x00000200
 #define ATOM_S7_DOS_MODE_NUMBERw1           0x0FFFF
 
 #define ATOM_S7_DOS_8BIT_DAC_EN_SHIFT       8
@@ -6242,6 +6343,7 @@ typedef struct _ATOM_MC_INIT_PARAM_TABLE
 #define _128Mx32            0x53
 #define _256Mx8             0x61
 #define _256Mx16            0x62
+#define _512Mx8             0x71
 
 #define SAMSUNG             0x1
 #define INFINEON            0x2
@@ -6987,9 +7089,10 @@ typedef struct _ATOM_DISP_OUT_INFO_V3
   UCHAR  ucMaxDispEngineNum;
   UCHAR  ucMaxActiveDispEngineNum;
   UCHAR  ucMaxPPLLNum;
-  UCHAR  ucCoreRefClkSource;                          // value of CORE_REF_CLK_SOURCE
-  UCHAR  ucReserved[3];
-        ASIC_TRANSMITTER_INFO_V2  asTransmitterInfo[1];     // for alligment only
+  UCHAR  ucCoreRefClkSource;                    // value of CORE_REF_CLK_SOURCE
+  UCHAR  ucDispCaps;
+  UCHAR  ucReserved[2];
+  ASIC_TRANSMITTER_INFO_V2  asTransmitterInfo[1];     // for alligment only
 }ATOM_DISP_OUT_INFO_V3;
 
 //ucDispCaps
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index bf87f6d435f8..80a20120e625 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -1753,7 +1753,7 @@ static int radeon_atom_pick_pll(struct drm_crtc *crtc)
                                 if (pll != ATOM_PPLL_INVALID)
                                         return pll;
                         }
-                } else {
+                } else if (!ASIC_IS_DCE41(rdev)) { /* Don't share PLLs on DCE4.1 chips */
                         /* use the same PPLL for all monitors with the same clock */
                         pll = radeon_get_shared_nondp_ppll(crtc);
                         if (pll != ATOM_PPLL_INVALID)
@@ -1910,6 +1910,21 @@ static void atombios_crtc_disable(struct drm_crtc *crtc)
         int i;
 
         atombios_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
+        if (crtc->fb) {
+                int r;
+                struct radeon_framebuffer *radeon_fb;
+                struct radeon_bo *rbo;
+
+                radeon_fb = to_radeon_framebuffer(crtc->fb);
+                rbo = gem_to_radeon_bo(radeon_fb->obj);
+                r = radeon_bo_reserve(rbo, false);
+                if (unlikely(r))
+                        DRM_ERROR("failed to reserve rbo before unpin\n");
+                else {
+                        radeon_bo_unpin(rbo);
+                        radeon_bo_unreserve(rbo);
+                }
+        }
         /* disable the GRPH */
         if (ASIC_IS_DCE4(rdev))
                 WREG32(EVERGREEN_GRPH_ENABLE + radeon_crtc->crtc_offset, 0);
@@ -1940,7 +1955,9 @@ static void atombios_crtc_disable(struct drm_crtc *crtc)
                 break;
         case ATOM_PPLL0:
                 /* disable the ppll */
-                if ((rdev->family == CHIP_ARUBA) || (rdev->family == CHIP_BONAIRE))
+                if ((rdev->family == CHIP_ARUBA) ||
+                    (rdev->family == CHIP_BONAIRE) ||
+                    (rdev->family == CHIP_HAWAII))
                         atombios_crtc_program_pll(crtc, radeon_crtc->crtc_id, radeon_crtc->pll_id,
                                                   0, 0, ATOM_DISABLE, 0, 0, 0, 0, 0, false, &ss);
                 break;
diff --git a/drivers/gpu/drm/radeon/atombios_encoders.c b/drivers/gpu/drm/radeon/atombios_encoders.c
index 5e891b226acf..a42d61571f49 100644
--- a/drivers/gpu/drm/radeon/atombios_encoders.c
+++ b/drivers/gpu/drm/radeon/atombios_encoders.c
@@ -213,7 +213,7 @@ void radeon_atom_backlight_init(struct radeon_encoder *radeon_encoder,
         props.type = BACKLIGHT_RAW;
         snprintf(bl_name, sizeof(bl_name),
                  "radeon_bl%d", dev->primary->index);
-        bd = backlight_device_register(bl_name, &drm_connector->kdev,
+        bd = backlight_device_register(bl_name, drm_connector->kdev,
                                        pdata, &radeon_atom_backlight_ops, &props);
         if (IS_ERR(bd)) {
                 DRM_ERROR("Backlight registration failed\n");
@@ -1662,19 +1662,11 @@ radeon_atom_encoder_dpms_dig(struct drm_encoder *encoder, int mode)
                         atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_SETUP, 0);
                         /* enable the transmitter */
                         atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE, 0, 0);
-                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE_OUTPUT, 0, 0);
                 } else {
                         /* setup and enable the encoder and transmitter */
                         atombios_dig_encoder_setup(encoder, ATOM_ENABLE, 0);
                         atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_SETUP, 0, 0);
                         atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE, 0, 0);
-                        /* some dce3.x boards have a bug in their transmitter control table.
-                         * ACTION_ENABLE_OUTPUT can probably be dropped since ACTION_ENABLE
-                         * does the same thing and more.
-                         */
-                        if ((rdev->family != CHIP_RV710) && (rdev->family != CHIP_RV730) &&
-                            (rdev->family != CHIP_RS780) && (rdev->family != CHIP_RS880))
-                                atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE_OUTPUT, 0, 0);
                 }
                 if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(encoder)) && connector) {
                         if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
@@ -1692,16 +1684,11 @@ radeon_atom_encoder_dpms_dig(struct drm_encoder *encoder, int mode)
         case DRM_MODE_DPMS_STANDBY:
         case DRM_MODE_DPMS_SUSPEND:
         case DRM_MODE_DPMS_OFF:
-                if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
-                        /* disable the transmitter */
-                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0);
-                } else if (ASIC_IS_DCE4(rdev)) {
+                if (ASIC_IS_DCE4(rdev)) {
                         /* disable the transmitter */
-                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE_OUTPUT, 0, 0);
                         atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0);
                 } else {
                         /* disable the encoder and transmitter */
-                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE_OUTPUT, 0, 0);
                         atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0);
                         atombios_dig_encoder_setup(encoder, ATOM_DISABLE, 0);
                 }
@@ -2410,6 +2397,15 @@ static void radeon_atom_encoder_prepare(struct drm_encoder *encoder)
 
         /* this is needed for the pll/ss setup to work correctly in some cases */
         atombios_set_encoder_crtc_source(encoder);
+        /* set up the FMT blocks */
+        if (ASIC_IS_DCE8(rdev))
+                dce8_program_fmt(encoder);
+        else if (ASIC_IS_DCE4(rdev))
+                dce4_program_fmt(encoder);
+        else if (ASIC_IS_DCE3(rdev))
+                dce3_program_fmt(encoder);
+        else if (ASIC_IS_AVIVO(rdev))
+                avivo_program_fmt(encoder);
 }
 
 static void radeon_atom_encoder_commit(struct drm_encoder *encoder)
diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c
index 51e947a97edf..1ed479976358 100644
--- a/drivers/gpu/drm/radeon/ci_dpm.c
+++ b/drivers/gpu/drm/radeon/ci_dpm.c
@@ -40,6 +40,20 @@
 #define VOLTAGE_VID_OFFSET_SCALE1    625
 #define VOLTAGE_VID_OFFSET_SCALE2    100
 
+static const struct ci_pt_defaults defaults_hawaii_xt =
+{
+        1, 0xF, 0xFD, 0x19, 5, 0x14, 0, 0xB0000,
+        { 0x84,  0x0,   0x0,   0x7F,  0x0,   0x0,   0x5A,  0x60,  0x51,  0x8E,  0x79,  0x6B,  0x5F,  0x90,  0x79  },
+        { 0x1EA, 0x1EA, 0x1EA, 0x224, 0x224, 0x224, 0x24F, 0x24F, 0x24F, 0x28E, 0x28E, 0x28E, 0x2BC, 0x2BC, 0x2BC }
+};
+
+static const struct ci_pt_defaults defaults_hawaii_pro =
+{
+        1, 0xF, 0xFD, 0x19, 5, 0x14, 0, 0x65062,
+        { 0x93,  0x0,   0x0,   0x97,  0x0,   0x0,   0x6B,  0x60,  0x51,  0x95,  0x79,  0x6B,  0x5F,  0x90,  0x79  },
+        { 0x1EA, 0x1EA, 0x1EA, 0x224, 0x224, 0x224, 0x24F, 0x24F, 0x24F, 0x28E, 0x28E, 0x28E, 0x2BC, 0x2BC, 0x2BC }
+};
+
 static const struct ci_pt_defaults defaults_bonaire_xt =
 {
         1, 0xF, 0xFD, 0x19, 5, 45, 0, 0xB0000,
@@ -187,22 +201,38 @@ static void ci_initialize_powertune_defaults(struct radeon_device *rdev)
         struct ci_power_info *pi = ci_get_pi(rdev);
 
         switch (rdev->pdev->device) {
-        case 0x6650:
-        case 0x6658:
-        case 0x665C:
-        default:
+        case 0x6650:
+        case 0x6658:
+        case 0x665C:
+        default:
                 pi->powertune_defaults = &defaults_bonaire_xt;
                 break;
-        case 0x6651:
-        case 0x665D:
+        case 0x6651:
+        case 0x665D:
                 pi->powertune_defaults = &defaults_bonaire_pro;
                 break;
-        case 0x6640:
+        case 0x6640:
                 pi->powertune_defaults = &defaults_saturn_xt;
                 break;
-        case 0x6641:
+        case 0x6641:
                 pi->powertune_defaults = &defaults_saturn_pro;
                 break;
+        case 0x67B8:
+        case 0x67B0:
+        case 0x67A0:
+        case 0x67A1:
+        case 0x67A2:
+        case 0x67A8:
+        case 0x67A9:
+        case 0x67AA:
+        case 0x67B9:
+        case 0x67BE:
+                pi->powertune_defaults = &defaults_hawaii_xt;
+                break;
+        case 0x67BA:
+        case 0x67B1:
+                pi->powertune_defaults = &defaults_hawaii_pro;
+                break;
         }
 
         pi->dte_tj_offset = 0;
@@ -5142,9 +5172,15 @@ int ci_dpm_init(struct radeon_device *rdev)
         rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
         rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
 
-        pi->thermal_temp_setting.temperature_low = 99500;
-        pi->thermal_temp_setting.temperature_high = 100000;
-        pi->thermal_temp_setting.temperature_shutdown = 104000;
+        if (rdev->family == CHIP_HAWAII) {
+                pi->thermal_temp_setting.temperature_low = 94500;
+                pi->thermal_temp_setting.temperature_high = 95000;
+                pi->thermal_temp_setting.temperature_shutdown = 104000;
+        } else {
+                pi->thermal_temp_setting.temperature_low = 99500;
+                pi->thermal_temp_setting.temperature_high = 100000;
+                pi->thermal_temp_setting.temperature_shutdown = 104000;
+        }
 
         pi->uvd_enabled = false;
 
diff --git a/drivers/gpu/drm/radeon/ci_smc.c b/drivers/gpu/drm/radeon/ci_smc.c
index 252e10a41cf5..9c745dd22438 100644
--- a/drivers/gpu/drm/radeon/ci_smc.c
+++ b/drivers/gpu/drm/radeon/ci_smc.c
@@ -217,6 +217,10 @@ int ci_load_smc_ucode(struct radeon_device *rdev, u32 limit)
                 ucode_start_address = BONAIRE_SMC_UCODE_START;
                 ucode_size = BONAIRE_SMC_UCODE_SIZE;
                 break;
+        case CHIP_HAWAII:
+                ucode_start_address = HAWAII_SMC_UCODE_START;
+                ucode_size = HAWAII_SMC_UCODE_SIZE;
+                break;
         default:
                 DRM_ERROR("unknown asic in smc ucode loader\n");
                 BUG();
diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c
index 9cd2bc989ac7..ae92aa041c6a 100644
--- a/drivers/gpu/drm/radeon/cik.c
+++ b/drivers/gpu/drm/radeon/cik.c
@@ -41,6 +41,14 @@ MODULE_FIRMWARE("radeon/BONAIRE_mc.bin");
 MODULE_FIRMWARE("radeon/BONAIRE_rlc.bin");
 MODULE_FIRMWARE("radeon/BONAIRE_sdma.bin");
 MODULE_FIRMWARE("radeon/BONAIRE_smc.bin");
+MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
+MODULE_FIRMWARE("radeon/HAWAII_me.bin");
+MODULE_FIRMWARE("radeon/HAWAII_ce.bin");
+MODULE_FIRMWARE("radeon/HAWAII_mec.bin");
+MODULE_FIRMWARE("radeon/HAWAII_mc.bin");
+MODULE_FIRMWARE("radeon/HAWAII_rlc.bin");
+MODULE_FIRMWARE("radeon/HAWAII_sdma.bin");
+MODULE_FIRMWARE("radeon/HAWAII_smc.bin");
 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
 MODULE_FIRMWARE("radeon/KAVERI_ce.bin");
@@ -67,11 +75,6 @@ extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
 extern int cik_sdma_resume(struct radeon_device *rdev);
 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
 extern void cik_sdma_fini(struct radeon_device *rdev);
-extern void cik_sdma_vm_set_page(struct radeon_device *rdev,
-                                 struct radeon_ib *ib,
-                                 uint64_t pe,
-                                 uint64_t addr, unsigned count,
-                                 uint32_t incr, uint32_t flags);
 static void cik_rlc_stop(struct radeon_device *rdev);
 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
 static void cik_program_aspm(struct radeon_device *rdev);
@@ -1302,6 +1305,171 @@ static const u32 kalindi_mgcg_cgcg_init[] =
         0xd80c, 0xff000ff0, 0x00000100
 };
 
+static const u32 hawaii_golden_spm_registers[] =
+{
+        0x30800, 0xe0ffffff, 0xe0000000
+};
+
+static const u32 hawaii_golden_common_registers[] =
+{
+        0x30800, 0xffffffff, 0xe0000000,
+        0x28350, 0xffffffff, 0x3a00161a,
+        0x28354, 0xffffffff, 0x0000002e,
+        0x9a10, 0xffffffff, 0x00018208,
+        0x98f8, 0xffffffff, 0x12011003
+};
+
+static const u32 hawaii_golden_registers[] =
+{
+        0x3354, 0x00000333, 0x00000333,
+        0x9a10, 0x00010000, 0x00058208,
+        0x9830, 0xffffffff, 0x00000000,
+        0x9834, 0xf00fffff, 0x00000400,
+        0x9838, 0x0002021c, 0x00020200,
+        0xc78, 0x00000080, 0x00000000,
+        0x5bb0, 0x000000f0, 0x00000070,
+        0x5bc0, 0xf0311fff, 0x80300000,
+        0x350c, 0x00810000, 0x408af000,
+        0x7030, 0x31000111, 0x00000011,
+        0x2f48, 0x73773777, 0x12010001,
+        0x2120, 0x0000007f, 0x0000001b,
+        0x21dc, 0x00007fb6, 0x00002191,
+        0x3628, 0x0000003f, 0x0000000a,
+        0x362c, 0x0000003f, 0x0000000a,
+        0x2ae4, 0x00073ffe, 0x000022a2,
+        0x240c, 0x000007ff, 0x00000000,
+        0x8bf0, 0x00002001, 0x00000001,
+        0x8b24, 0xffffffff, 0x00ffffff,
+        0x30a04, 0x0000ff0f, 0x00000000,
+        0x28a4c, 0x07ffffff, 0x06000000,
+        0x3e78, 0x00000001, 0x00000002,
+        0xc768, 0x00000008, 0x00000008,
+        0xc770, 0x00000f00, 0x00000800,
+        0xc774, 0x00000f00, 0x00000800,
+        0xc798, 0x00ffffff, 0x00ff7fbf,
+        0xc79c, 0x00ffffff, 0x00ff7faf,
+        0x8c00, 0x000000ff, 0x00000800,
+        0xe40, 0x00001fff, 0x00001fff,
+        0x9060, 0x0000007f, 0x00000020,
+        0x9508, 0x00010000, 0x00010000,
+        0xae00, 0x00100000, 0x000ff07c,
+        0xac14, 0x000003ff, 0x0000000f,
+        0xac10, 0xffffffff, 0x7564fdec,
+        0xac0c, 0xffffffff, 0x3120b9a8,
+        0xac08, 0x20000000, 0x0f9c0000
+};
+
+static const u32 hawaii_mgcg_cgcg_init[] =
+{
+        0xc420, 0xffffffff, 0xfffffffd,
+        0x30800, 0xffffffff, 0xe0000000,
+        0x3c2a0, 0xffffffff, 0x00000100,
+        0x3c208, 0xffffffff, 0x00000100,
+        0x3c2c0, 0xffffffff, 0x00000100,
+        0x3c2c8, 0xffffffff, 0x00000100,
+        0x3c2c4, 0xffffffff, 0x00000100,
+        0x55e4, 0xffffffff, 0x00200100,
+        0x3c280, 0xffffffff, 0x00000100,
+        0x3c214, 0xffffffff, 0x06000100,
+        0x3c220, 0xffffffff, 0x00000100,
+        0x3c218, 0xffffffff, 0x06000100,
+        0x3c204, 0xffffffff, 0x00000100,
+        0x3c2e0, 0xffffffff, 0x00000100,
+        0x3c224, 0xffffffff, 0x00000100,
+        0x3c200, 0xffffffff, 0x00000100,
+        0x3c230, 0xffffffff, 0x00000100,
+        0x3c234, 0xffffffff, 0x00000100,
+        0x3c250, 0xffffffff, 0x00000100,
+        0x3c254, 0xffffffff, 0x00000100,
+        0x3c258, 0xffffffff, 0x00000100,
+        0x3c25c, 0xffffffff, 0x00000100,
+        0x3c260, 0xffffffff, 0x00000100,
+        0x3c27c, 0xffffffff, 0x00000100,
+        0x3c278, 0xffffffff, 0x00000100,
+        0x3c210, 0xffffffff, 0x06000100,
+        0x3c290, 0xffffffff, 0x00000100,
+        0x3c274, 0xffffffff, 0x00000100,
+        0x3c2b4, 0xffffffff, 0x00000100,
+        0x3c2b0, 0xffffffff, 0x00000100,
+        0x3c270, 0xffffffff, 0x00000100,
+        0x30800, 0xffffffff, 0xe0000000,
+        0x3c020, 0xffffffff, 0x00010000,
+        0x3c024, 0xffffffff, 0x00030002,
+        0x3c028, 0xffffffff, 0x00040007,
+        0x3c02c, 0xffffffff, 0x00060005,
+        0x3c030, 0xffffffff, 0x00090008,
+        0x3c034, 0xffffffff, 0x00010000,
+        0x3c038, 0xffffffff, 0x00030002,
+        0x3c03c, 0xffffffff, 0x00040007,
+        0x3c040, 0xffffffff, 0x00060005,
+        0x3c044, 0xffffffff, 0x00090008,
+        0x3c048, 0xffffffff, 0x00010000,
+        0x3c04c, 0xffffffff, 0x00030002,
+        0x3c050, 0xffffffff, 0x00040007,
+        0x3c054, 0xffffffff, 0x00060005,
+        0x3c058, 0xffffffff, 0x00090008,
+        0x3c05c, 0xffffffff, 0x00010000,
+        0x3c060, 0xffffffff, 0x00030002,
+        0x3c064, 0xffffffff, 0x00040007,
+        0x3c068, 0xffffffff, 0x00060005,
+        0x3c06c, 0xffffffff, 0x00090008,
+        0x3c070, 0xffffffff, 0x00010000,
+        0x3c074, 0xffffffff, 0x00030002,
+        0x3c078, 0xffffffff, 0x00040007,
+        0x3c07c, 0xffffffff, 0x00060005,
+        0x3c080, 0xffffffff, 0x00090008,
+        0x3c084, 0xffffffff, 0x00010000,
+        0x3c088, 0xffffffff, 0x00030002,
+        0x3c08c, 0xffffffff, 0x00040007,
+        0x3c090, 0xffffffff, 0x00060005,
+        0x3c094, 0xffffffff, 0x00090008,
+        0x3c098, 0xffffffff, 0x00010000,
+        0x3c09c, 0xffffffff, 0x00030002,
+        0x3c0a0, 0xffffffff, 0x00040007,
+        0x3c0a4, 0xffffffff, 0x00060005,
+        0x3c0a8, 0xffffffff, 0x00090008,
+        0x3c0ac, 0xffffffff, 0x00010000,
+        0x3c0b0, 0xffffffff, 0x00030002,
+        0x3c0b4, 0xffffffff, 0x00040007,
+        0x3c0b8, 0xffffffff, 0x00060005,
+        0x3c0bc, 0xffffffff, 0x00090008,
+        0x3c0c0, 0xffffffff, 0x00010000,
+        0x3c0c4, 0xffffffff, 0x00030002,
+        0x3c0c8, 0xffffffff, 0x00040007,
+        0x3c0cc, 0xffffffff, 0x00060005,
+        0x3c0d0, 0xffffffff, 0x00090008,
+        0x3c0d4, 0xffffffff, 0x00010000,
+        0x3c0d8, 0xffffffff, 0x00030002,
+        0x3c0dc, 0xffffffff, 0x00040007,
+        0x3c0e0, 0xffffffff, 0x00060005,
+        0x3c0e4, 0xffffffff, 0x00090008,
+        0x3c0e8, 0xffffffff, 0x00010000,
+        0x3c0ec, 0xffffffff, 0x00030002,
+        0x3c0f0, 0xffffffff, 0x00040007,
+        0x3c0f4, 0xffffffff, 0x00060005,
+        0x3c0f8, 0xffffffff, 0x00090008,
+        0xc318, 0xffffffff, 0x00020200,
+        0x3350, 0xffffffff, 0x00000200,
+        0x15c0, 0xffffffff, 0x00000400,
+        0x55e8, 0xffffffff, 0x00000000,
+        0x2f50, 0xffffffff, 0x00000902,
+        0x3c000, 0xffffffff, 0x96940200,
+        0x8708, 0xffffffff, 0x00900100,
+        0xc424, 0xffffffff, 0x0020003f,
+        0x38, 0xffffffff, 0x0140001c,
+        0x3c, 0x000f0000, 0x000f0000,
+        0x220, 0xffffffff, 0xc060000c,
+        0x224, 0xc0000fff, 0x00000100,
+        0xf90, 0xffffffff, 0x00000100,
+        0xf98, 0x00000101, 0x00000000,
+        0x20a8, 0xffffffff, 0x00000104,
+        0x55e4, 0xff000fff, 0x00000100,
+        0x30cc, 0xc0000fff, 0x00000104,
+        0xc1e4, 0x00000001, 0x00000001,
+        0xd00c, 0xff000ff0, 0x00000100,
+        0xd80c, 0xff000ff0, 0x00000100
+};
+
 static void cik_init_golden_registers(struct radeon_device *rdev)
 {
         switch (rdev->family) {
@@ -1347,6 +1515,20 @@ static void cik_init_golden_registers(struct radeon_device *rdev)
                                                  spectre_golden_spm_registers,
                                                  (const u32)ARRAY_SIZE(spectre_golden_spm_registers));
                 break;
+        case CHIP_HAWAII:
+                radeon_program_register_sequence(rdev,
+                                                 hawaii_mgcg_cgcg_init,
+                                                 (const u32)ARRAY_SIZE(hawaii_mgcg_cgcg_init));
+                radeon_program_register_sequence(rdev,
+                                                 hawaii_golden_registers,
+                                                 (const u32)ARRAY_SIZE(hawaii_golden_registers));
+                radeon_program_register_sequence(rdev,
+                                                 hawaii_golden_common_registers,
+                                                 (const u32)ARRAY_SIZE(hawaii_golden_common_registers));
+                radeon_program_register_sequence(rdev,
+                                                 hawaii_golden_spm_registers,
+                                                 (const u32)ARRAY_SIZE(hawaii_golden_spm_registers));
+                break;
         default:
                 break;
         }
@@ -1454,6 +1636,35 @@ static const u32 bonaire_io_mc_regs[BONAIRE_IO_MC_REGS_SIZE][2] =
         {0x0000009f, 0x00b48000}
 };
 
+#define HAWAII_IO_MC_REGS_SIZE 22
+
+static const u32 hawaii_io_mc_regs[HAWAII_IO_MC_REGS_SIZE][2] =
+{
+        {0x0000007d, 0x40000000},
+        {0x0000007e, 0x40180304},
+        {0x0000007f, 0x0000ff00},
+        {0x00000081, 0x00000000},
+        {0x00000083, 0x00000800},
+        {0x00000086, 0x00000000},
+        {0x00000087, 0x00000100},
+        {0x00000088, 0x00020100},
+        {0x00000089, 0x00000000},
+        {0x0000008b, 0x00040000},
+        {0x0000008c, 0x00000100},
+        {0x0000008e, 0xff010000},
+        {0x00000090, 0xffffefff},
+        {0x00000091, 0xfff3efff},
+        {0x00000092, 0xfff3efbf},
+        {0x00000093, 0xf7ffffff},
+        {0x00000094, 0xffffff7f},
+        {0x00000095, 0x00000fff},
+        {0x00000096, 0x00116fff},
+        {0x00000097, 0x60010000},
+        {0x00000098, 0x10010000},
+        {0x0000009f, 0x00c79000}
+};
+
+
 /**
  * cik_srbm_select - select specific register instances
  *
@@ -1498,11 +1709,17 @@ static int ci_mc_load_microcode(struct radeon_device *rdev)
 
         switch (rdev->family) {
         case CHIP_BONAIRE:
-        default:
                 io_mc_regs = (u32 *)&bonaire_io_mc_regs;
                 ucode_size = CIK_MC_UCODE_SIZE;
                 regs_size = BONAIRE_IO_MC_REGS_SIZE;
                 break;
+        case CHIP_HAWAII:
+                io_mc_regs = (u32 *)&hawaii_io_mc_regs;
+                ucode_size = HAWAII_MC_UCODE_SIZE;
+                regs_size = HAWAII_IO_MC_REGS_SIZE;
+                break;
+        default:
+                return -EINVAL;
         }
 
         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
@@ -1564,8 +1781,8 @@ static int cik_init_microcode(struct radeon_device *rdev)
 {
         const char *chip_name;
         size_t pfp_req_size, me_req_size, ce_req_size,
-                mec_req_size, rlc_req_size, mc_req_size,
-                sdma_req_size, smc_req_size;
+                mec_req_size, rlc_req_size, mc_req_size = 0,
+                sdma_req_size, smc_req_size = 0;
         char fw_name[30];
         int err;
 
@@ -1583,6 +1800,17 @@ static int cik_init_microcode(struct radeon_device *rdev)
                 sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
                 smc_req_size = ALIGN(BONAIRE_SMC_UCODE_SIZE, 4);
                 break;
+        case CHIP_HAWAII:
+                chip_name = "HAWAII";
+                pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
+                me_req_size = CIK_ME_UCODE_SIZE * 4;
+                ce_req_size = CIK_CE_UCODE_SIZE * 4;
+                mec_req_size = CIK_MEC_UCODE_SIZE * 4;
+                rlc_req_size = BONAIRE_RLC_UCODE_SIZE * 4;
+                mc_req_size = HAWAII_MC_UCODE_SIZE * 4;
+                sdma_req_size = CIK_SDMA_UCODE_SIZE * 4;
+                smc_req_size = ALIGN(HAWAII_SMC_UCODE_SIZE, 4);
+                break;
         case CHIP_KAVERI:
                 chip_name = "KAVERI";
                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
@@ -1763,9 +1991,227 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
 
         num_pipe_configs = rdev->config.cik.max_tile_pipes;
         if (num_pipe_configs > 8)
-                num_pipe_configs = 8; /* ??? */
+                num_pipe_configs = 16;
 
-        if (num_pipe_configs == 8) {
+        if (num_pipe_configs == 16) {
+                for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
+                        switch (reg_offset) {
+                        case 0:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B));
+                                break;
+                        case 1:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B));
+                                break;
+                        case 2:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
+                                break;
+                        case 3:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B));
+                                break;
+                        case 4:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+                                                 TILE_SPLIT(split_equal_to_row_size));
+                                break;
+                        case 5:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
+                                break;
+                        case 6:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+                                                 TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B));
+                                break;
+                        case 7:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+                                                 TILE_SPLIT(split_equal_to_row_size));
+                                break;
+                        case 8:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16));
+                                break;
+                        case 9:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
+                                break;
+                        case 10:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 11:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 12:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 13:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
+                                break;
+                        case 14:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 16:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 17:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 27:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
+                                break;
+                        case 28:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 29:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_8x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        case 30:
+                                gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
+                                                 SAMPLE_SPLIT(ADDR_SURF_SAMPLE_SPLIT_2));
+                                break;
+                        default:
+                                gb_tile_moden = 0;
+                                break;
+                        }
+                        rdev->config.cik.tile_mode_array[reg_offset] = gb_tile_moden;
+                        WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
+                }
+                for (reg_offset = 0; reg_offset < num_secondary_tile_mode_states; reg_offset++) {
+                        switch (reg_offset) {
+                        case 0:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 1:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 2:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 3:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 4:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_8_BANK));
+                                break;
+                        case 5:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_4_BANK));
+                                break;
+                        case 6:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_2_BANK));
+                                break;
+                        case 8:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 9:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 10:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_16_BANK));
+                                break;
+                        case 11:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_8_BANK));
+                                break;
+                        case 12:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_4_BANK));
+                                break;
+                        case 13:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_2_BANK));
+                                break;
+                        case 14:
+                                gb_tile_moden = (BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
+                                                 BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
+                                                 MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1) |
+                                                 NUM_BANKS(ADDR_SURF_2_BANK));
+                                break;
+                        default:
+                                gb_tile_moden = 0;
+                                break;
+                        }
+                        WREG32(GB_MACROTILE_MODE0 + (reg_offset * 4), gb_tile_moden);
+                }
+        } else if (num_pipe_configs == 8) {
                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
                         switch (reg_offset) {
                         case 0:
@@ -2650,7 +3096,10 @@ static void cik_setup_rb(struct radeon_device *rdev,
                 for (j = 0; j < sh_per_se; j++) {
                         cik_select_se_sh(rdev, i, j);
                         data = cik_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
-                        disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
+                        if (rdev->family == CHIP_HAWAII)
+                                disabled_rbs |= data << ((i * sh_per_se + j) * HAWAII_RB_BITMAP_WIDTH_PER_SH);
+                        else
+                                disabled_rbs |= data << ((i * sh_per_se + j) * CIK_RB_BITMAP_WIDTH_PER_SH);
                 }
         }
         cik_select_se_sh(rdev, 0xffffffff, 0xffffffff);
@@ -2667,6 +3116,12 @@ static void cik_setup_rb(struct radeon_device *rdev,
                 data = 0;
                 for (j = 0; j < sh_per_se; j++) {
                         switch (enabled_rbs & 3) {
+                        case 0:
+                                if (j == 0)
+                                        data |= PKR_MAP(RASTER_CONFIG_RB_MAP_3);
+                                else
+                                        data |= PKR_MAP(RASTER_CONFIG_RB_MAP_0);
+                                break;
                         case 1:
                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
                                 break;
@@ -2719,6 +3174,23 @@ static void cik_gpu_init(struct radeon_device *rdev)
                 rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
                 gb_addr_config = BONAIRE_GB_ADDR_CONFIG_GOLDEN;
                 break;
+        case CHIP_HAWAII:
+                rdev->config.cik.max_shader_engines = 4;
+                rdev->config.cik.max_tile_pipes = 16;
+                rdev->config.cik.max_cu_per_sh = 11;
+                rdev->config.cik.max_sh_per_se = 1;
+                rdev->config.cik.max_backends_per_se = 4;
+                rdev->config.cik.max_texture_channel_caches = 16;
+                rdev->config.cik.max_gprs = 256;
+                rdev->config.cik.max_gs_threads = 32;
+                rdev->config.cik.max_hw_contexts = 8;
+
+                rdev->config.cik.sc_prim_fifo_size_frontend = 0x20;
+                rdev->config.cik.sc_prim_fifo_size_backend = 0x100;
+                rdev->config.cik.sc_hiz_tile_fifo_size = 0x30;
+                rdev->config.cik.sc_earlyz_tile_fifo_size = 0x130;
+                gb_addr_config = HAWAII_GB_ADDR_CONFIG_GOLDEN;
+                break;
         case CHIP_KAVERI:
                 rdev->config.cik.max_shader_engines = 1;
                 rdev->config.cik.max_tile_pipes = 4;
@@ -3097,6 +3569,85 @@ void cik_semaphore_ring_emit(struct radeon_device *rdev,
         radeon_ring_write(ring, (upper_32_bits(addr) & 0xffff) | sel);
 }
 
+/**
+ * cik_copy_cpdma - copy pages using the CP DMA engine
+ *
+ * @rdev: radeon_device pointer
+ * @src_offset: src GPU address
+ * @dst_offset: dst GPU address
+ * @num_gpu_pages: number of GPU pages to xfer
+ * @fence: radeon fence object
+ *
+ * Copy GPU paging using the CP DMA engine (CIK+).
+ * Used by the radeon ttm implementation to move pages if
+ * registered as the asic copy callback.
+ */
+int cik_copy_cpdma(struct radeon_device *rdev,
+                   uint64_t src_offset, uint64_t dst_offset,
+                   unsigned num_gpu_pages,
+                   struct radeon_fence **fence)
+{
+        struct radeon_semaphore *sem = NULL;
+        int ring_index = rdev->asic->copy.blit_ring_index;
+        struct radeon_ring *ring = &rdev->ring[ring_index];
+        u32 size_in_bytes, cur_size_in_bytes, control;
+        int i, num_loops;
+        int r = 0;
+
+        r = radeon_semaphore_create(rdev, &sem);
+        if (r) {
+                DRM_ERROR("radeon: moving bo (%d).\n", r);
+                return r;
+        }
+
+        size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
+        num_loops = DIV_ROUND_UP(size_in_bytes, 0x1fffff);
+        r = radeon_ring_lock(rdev, ring, num_loops * 7 + 18);
+        if (r) {
+                DRM_ERROR("radeon: moving bo (%d).\n", r);
+                radeon_semaphore_free(rdev, &sem, NULL);
+                return r;
+        }
+
+        if (radeon_fence_need_sync(*fence, ring->idx)) {
+                radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
+                                            ring->idx);
+                radeon_fence_note_sync(*fence, ring->idx);
+        } else {
+                radeon_semaphore_free(rdev, &sem, NULL);
+        }
+
+        for (i = 0; i < num_loops; i++) {
+                cur_size_in_bytes = size_in_bytes;
+                if (cur_size_in_bytes > 0x1fffff)
+                        cur_size_in_bytes = 0x1fffff;
+                size_in_bytes -= cur_size_in_bytes;
+                control = 0;
+                if (size_in_bytes == 0)
+                        control |= PACKET3_DMA_DATA_CP_SYNC;
+                radeon_ring_write(ring, PACKET3(PACKET3_DMA_DATA, 5));
+                radeon_ring_write(ring, control);
+                radeon_ring_write(ring, lower_32_bits(src_offset));
+                radeon_ring_write(ring, upper_32_bits(src_offset));
+                radeon_ring_write(ring, lower_32_bits(dst_offset));
+                radeon_ring_write(ring, upper_32_bits(dst_offset));
+                radeon_ring_write(ring, cur_size_in_bytes);
+                src_offset += cur_size_in_bytes;
+                dst_offset += cur_size_in_bytes;
+        }
+
+        r = radeon_fence_emit(rdev, fence, ring->idx);
+        if (r) {
+                radeon_ring_unlock_undo(rdev, ring);
+                return r;
+        }
+
+        radeon_ring_unlock_commit(rdev, ring);
+        radeon_semaphore_free(rdev, &sem, *fence);
+
+        return r;
+}
+
 /*
  * IB stuff
  */
@@ -3403,7 +3954,8 @@ static int cik_cp_gfx_resume(struct radeon_device *rdev)
         int r;
 
         WREG32(CP_SEM_WAIT_TIMER, 0x0);
-        WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
+        if (rdev->family != CHIP_HAWAII)
+                WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
 
         /* Set the write pointer delay */
         WREG32(CP_RB_WPTR_DELAY, 0);
@@ -4740,12 +5292,17 @@ void cik_vm_fini(struct radeon_device *rdev)
 static void cik_vm_decode_fault(struct radeon_device *rdev,
                                 u32 status, u32 addr, u32 mc_client)
 {
-        u32 mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
+        u32 mc_id;
         u32 vmid = (status & FAULT_VMID_MASK) >> FAULT_VMID_SHIFT;
         u32 protections = (status & PROTECTIONS_MASK) >> PROTECTIONS_SHIFT;
         char block[5] = { mc_client >> 24, (mc_client >> 16) & 0xff,
                 (mc_client >> 8) & 0xff, mc_client & 0xff, 0 };
 
+        if (rdev->family == CHIP_HAWAII)
+                mc_id = (status & HAWAII_MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
+        else
+                mc_id = (status & MEMORY_CLIENT_ID_MASK) >> MEMORY_CLIENT_ID_SHIFT;
+
         printk("VM fault (0x%02x, vmid %d) at page %u, %s from '%s' (0x%08x) (%d)\n",
                protections, vmid, addr,
                (status & MEMORY_CLIENT_RW_MASK) ? "write" : "read",
@@ -4834,62 +5391,6 @@ void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
         }
 }
 
-/**
- * cik_vm_set_page - update the page tables using sDMA
- *
- * @rdev: radeon_device pointer
- * @ib: indirect buffer to fill with commands
- * @pe: addr of the page entry
- * @addr: dst addr to write into pe
- * @count: number of page entries to update
- * @incr: increase next addr by incr bytes
- * @flags: access flags
- *
- * Update the page tables using CP or sDMA (CIK).
- */
-void cik_vm_set_page(struct radeon_device *rdev,
-                     struct radeon_ib *ib,
-                     uint64_t pe,
-                     uint64_t addr, unsigned count,
-                     uint32_t incr, uint32_t flags)
-{
-        uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
-        uint64_t value;
-        unsigned ndw;
-
-        if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
-                /* CP */
-                while (count) {
-                        ndw = 2 + count * 2;
-                        if (ndw > 0x3FFE)
-                                ndw = 0x3FFE;
-
-                        ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
-                        ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
-                                                    WRITE_DATA_DST_SEL(1));
-                        ib->ptr[ib->length_dw++] = pe;
-                        ib->ptr[ib->length_dw++] = upper_32_bits(pe);
-                        for (; ndw > 2; ndw -= 2, --count, pe += 8) {
-                                if (flags & RADEON_VM_PAGE_SYSTEM) {
-                                        value = radeon_vm_map_gart(rdev, addr);
-                                        value &= 0xFFFFFFFFFFFFF000ULL;
-                                } else if (flags & RADEON_VM_PAGE_VALID) {
-                                        value = addr;
-                                } else {
-                                        value = 0;
-                                }
-                                addr += incr;
-                                value |= r600_flags;
-                                ib->ptr[ib->length_dw++] = value;
-                                ib->ptr[ib->length_dw++] = upper_32_bits(value);
-                        }
-                }
-        } else {
-                /* DMA */
-                cik_sdma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
-        }
-}
-
 /*
  * RLC
  * The RLC is a multi-purpose microengine that handles a
@@ -5058,6 +5559,7 @@ static int cik_rlc_resume(struct radeon_device *rdev)
 
         switch (rdev->family) {
         case CHIP_BONAIRE:
+        case CHIP_HAWAII:
         default:
                 size = BONAIRE_RLC_UCODE_SIZE;
                 break;
@@ -5556,7 +6058,7 @@ void cik_init_cp_pg_table(struct radeon_device *rdev)
                 }
 
                 for (i = 0; i < CP_ME_TABLE_SIZE; i ++) {
-                        dst_ptr[bo_offset + i] = be32_to_cpu(fw_data[table_offset + i]);
+                        dst_ptr[bo_offset + i] = cpu_to_le32(be32_to_cpu(fw_data[table_offset + i]));
                 }
                 bo_offset += CP_ME_TABLE_SIZE;
         }
@@ -5778,52 +6280,57 @@ void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
         if (buffer == NULL)
                 return;
 
-        buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
-        buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE;
+        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+        buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
 
-        buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1);
-        buffer[count++] = 0x80000000;
-        buffer[count++] = 0x80000000;
+        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
+        buffer[count++] = cpu_to_le32(0x80000000);
+        buffer[count++] = cpu_to_le32(0x80000000);
 
         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
                 for (ext = sect->section; ext->extent != NULL; ++ext) {
                         if (sect->id == SECT_CONTEXT) {
-                                buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count);
-                                buffer[count++] = ext->reg_index - 0xa000;
+                                buffer[count++] =
+                                        cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
+                                buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
                                 for (i = 0; i < ext->reg_count; i++)
-                                        buffer[count++] = ext->extent[i];
+                                        buffer[count++] = cpu_to_le32(ext->extent[i]);
                         } else {
                                 return;
                         }
                 }
         }
 
-        buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 2);
-        buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START;
+        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 2));
+        buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
         switch (rdev->family) {
         case CHIP_BONAIRE:
-                buffer[count++] = 0x16000012;
-                buffer[count++] = 0x00000000;
+                buffer[count++] = cpu_to_le32(0x16000012);
+                buffer[count++] = cpu_to_le32(0x00000000);
                 break;
         case CHIP_KAVERI:
-                buffer[count++] = 0x00000000; /* XXX */
-                buffer[count++] = 0x00000000;
+                buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
+                buffer[count++] = cpu_to_le32(0x00000000);
                 break;
         case CHIP_KABINI:
-                buffer[count++] = 0x00000000; /* XXX */
-                buffer[count++] = 0x00000000;
+                buffer[count++] = cpu_to_le32(0x00000000); /* XXX */
+                buffer[count++] = cpu_to_le32(0x00000000);
+                break;
+        case CHIP_HAWAII:
+                buffer[count++] = 0x3a00161a;
+                buffer[count++] = 0x0000002e;
                 break;
         default:
-                buffer[count++] = 0x00000000;
-                buffer[count++] = 0x00000000;
+                buffer[count++] = cpu_to_le32(0x00000000);
+                buffer[count++] = cpu_to_le32(0x00000000);
                 break;
         }
 
-        buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
-        buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE;
+        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+        buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
 
-        buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0);
-        buffer[count++] = 0;
+        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
+        buffer[count++] = cpu_to_le32(0);
 }
 
 static void cik_init_pg(struct radeon_device *rdev)
@@ -7118,7 +7625,7 @@ static int cik_startup(struct radeon_device *rdev)
         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
                              CP_RB0_RPTR, CP_RB0_WPTR,
-                             RADEON_CP_PACKET2);
+                             PACKET3(PACKET3_NOP, 0x3FFF));
         if (r)
                 return r;
 
@@ -7428,6 +7935,70 @@ void cik_fini(struct radeon_device *rdev)
         rdev->bios = NULL;
 }
 
+void dce8_program_fmt(struct drm_encoder *encoder)
+{
+        struct drm_device *dev = encoder->dev;
+        struct radeon_device *rdev = dev->dev_private;
+        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+        struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
+        struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
+        int bpc = 0;
+        u32 tmp = 0;
+        enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
+
+        if (connector) {
+                struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+                bpc = radeon_get_monitor_bpc(connector);
+                dither = radeon_connector->dither;
+        }
+
+        /* LVDS/eDP FMT is set up by atom */
+        if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
+                return;
+
+        /* not needed for analog */
+        if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
+            (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
+                return;
+
+        if (bpc == 0)
+                return;
+
+        switch (bpc) {
+        case 6:
+                if (dither == RADEON_FMT_DITHER_ENABLE)
+                        /* XXX sort out optimal dither settings */
+                        tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
+                                FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(0));
+                else
+                        tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(0));
+                break;
+        case 8:
+                if (dither == RADEON_FMT_DITHER_ENABLE)
+                        /* XXX sort out optimal dither settings */
+                        tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
+                                FMT_RGB_RANDOM_ENABLE |
+                                FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(1));
+                else
+                        tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(1));
+                break;
+        case 10:
+                if (dither == RADEON_FMT_DITHER_ENABLE)
+                        /* XXX sort out optimal dither settings */
+                        tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
+                                FMT_RGB_RANDOM_ENABLE |
+                                FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH(2));
+                else
+                        tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH(2));
+                break;
+        default:
+                /* not needed */
+                break;
+        }
+
+        WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
+}
+
 /* display watermark setup */
 /**
  * dce8_line_buffer_adjust - Set up the line buffer
diff --git a/drivers/gpu/drm/radeon/cik_sdma.c b/drivers/gpu/drm/radeon/cik_sdma.c
index b6286068e111..9c9529de20ee 100644
--- a/drivers/gpu/drm/radeon/cik_sdma.c
+++ b/drivers/gpu/drm/radeon/cik_sdma.c
@@ -25,6 +25,7 @@
 #include <drm/drmP.h>
 #include "radeon.h"
 #include "radeon_asic.h"
+#include "radeon_trace.h"
 #include "cikd.h"
 
 /* sdma */
@@ -101,14 +102,6 @@ void cik_sdma_fence_ring_emit(struct radeon_device *rdev,
 {
         struct radeon_ring *ring = &rdev->ring[fence->ring];
         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
-        u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
-                          SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
-        u32 ref_and_mask;
-
-        if (fence->ring == R600_RING_TYPE_DMA_INDEX)
-                ref_and_mask = SDMA0;
-        else
-                ref_and_mask = SDMA1;
 
         /* write the fence */
         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_FENCE, 0, 0));
@@ -118,12 +111,12 @@ void cik_sdma_fence_ring_emit(struct radeon_device *rdev,
         /* generate an interrupt */
         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_TRAP, 0, 0));
         /* flush HDP */
-        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
-        radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
-        radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
-        radeon_ring_write(ring, ref_and_mask); /* REFERENCE */
-        radeon_ring_write(ring, ref_and_mask); /* MASK */
-        radeon_ring_write(ring, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
+        /* We should be using the new POLL_REG_MEM special op packet here
+         * but it causes sDMA to hang sometimes
+         */
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
+        radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
+        radeon_ring_write(ring, 0);
 }
 
 /**
@@ -653,11 +646,12 @@ void cik_sdma_vm_set_page(struct radeon_device *rdev,
                           uint64_t addr, unsigned count,
                           uint32_t incr, uint32_t flags)
 {
-        uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
         uint64_t value;
         unsigned ndw;
 
-        if (flags & RADEON_VM_PAGE_SYSTEM) {
+        trace_radeon_vm_set_page(pe, addr, count, incr, flags);
+
+        if (flags & R600_PTE_SYSTEM) {
                 while (count) {
                         ndw = count * 2;
                         if (ndw > 0xFFFFE)
@@ -669,16 +663,10 @@ void cik_sdma_vm_set_page(struct radeon_device *rdev,
                         ib->ptr[ib->length_dw++] = upper_32_bits(pe);
                         ib->ptr[ib->length_dw++] = ndw;
                         for (; ndw > 0; ndw -= 2, --count, pe += 8) {
-                                if (flags & RADEON_VM_PAGE_SYSTEM) {
-                                        value = radeon_vm_map_gart(rdev, addr);
-                                        value &= 0xFFFFFFFFFFFFF000ULL;
-                                } else if (flags & RADEON_VM_PAGE_VALID) {
-                                        value = addr;
-                                } else {
-                                        value = 0;
-                                }
+                                value = radeon_vm_map_gart(rdev, addr);
+                                value &= 0xFFFFFFFFFFFFF000ULL;
                                 addr += incr;
-                                value |= r600_flags;
+                                value |= flags;
                                 ib->ptr[ib->length_dw++] = value;
                                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
                         }
@@ -689,7 +677,7 @@ void cik_sdma_vm_set_page(struct radeon_device *rdev,
                         if (ndw > 0x7FFFF)
                                 ndw = 0x7FFFF;
 
-                        if (flags & RADEON_VM_PAGE_VALID)
+                        if (flags & R600_PTE_VALID)
                                 value = addr;
                         else
                                 value = 0;
@@ -697,7 +685,7 @@ void cik_sdma_vm_set_page(struct radeon_device *rdev,
                         ib->ptr[ib->length_dw++] = SDMA_PACKET(SDMA_OPCODE_GENERATE_PTE_PDE, 0, 0);
                         ib->ptr[ib->length_dw++] = pe; /* dst addr */
                         ib->ptr[ib->length_dw++] = upper_32_bits(pe);
-                        ib->ptr[ib->length_dw++] = r600_flags; /* mask */
+                        ib->ptr[ib->length_dw++] = flags; /* mask */
                         ib->ptr[ib->length_dw++] = 0;
                         ib->ptr[ib->length_dw++] = value; /* value */
                         ib->ptr[ib->length_dw++] = upper_32_bits(value);
@@ -724,18 +712,10 @@ void cik_sdma_vm_set_page(struct radeon_device *rdev,
 void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
 {
         struct radeon_ring *ring = &rdev->ring[ridx];
-        u32 extra_bits = (SDMA_POLL_REG_MEM_EXTRA_OP(1) |
-                          SDMA_POLL_REG_MEM_EXTRA_FUNC(3)); /* == */
-        u32 ref_and_mask;
 
         if (vm == NULL)
                 return;
 
-        if (ridx == R600_RING_TYPE_DMA_INDEX)
-                ref_and_mask = SDMA0;
-        else
-                ref_and_mask = SDMA1;
-
         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
         if (vm->id < 8) {
                 radeon_ring_write(ring, (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
@@ -770,12 +750,12 @@ void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm
         radeon_ring_write(ring, VMID(0));
 
         /* flush HDP */
-        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_POLL_REG_MEM, 0, extra_bits));
-        radeon_ring_write(ring, GPU_HDP_FLUSH_DONE);
-        radeon_ring_write(ring, GPU_HDP_FLUSH_REQ);
-        radeon_ring_write(ring, ref_and_mask); /* REFERENCE */
-        radeon_ring_write(ring, ref_and_mask); /* MASK */
-        radeon_ring_write(ring, (4 << 16) | 10); /* RETRY_COUNT, POLL_INTERVAL */
+        /* We should be using the new POLL_REG_MEM special op packet here
+         * but it causes sDMA to hang sometimes
+         */
+        radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
+        radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
+        radeon_ring_write(ring, 0);
 
         /* flush TLB */
         radeon_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_SRBM_WRITE, 0, 0xf000));
diff --git a/drivers/gpu/drm/radeon/cikd.h b/drivers/gpu/drm/radeon/cikd.h
index 203d2a09a1f5..5964af5e5b2d 100644
--- a/drivers/gpu/drm/radeon/cikd.h
+++ b/drivers/gpu/drm/radeon/cikd.h
@@ -25,8 +25,10 @@
 #define CIK_H
 
 #define BONAIRE_GB_ADDR_CONFIG_GOLDEN        0x12010001
+#define HAWAII_GB_ADDR_CONFIG_GOLDEN         0x12011003
 
-#define CIK_RB_BITMAP_WIDTH_PER_SH  2
+#define CIK_RB_BITMAP_WIDTH_PER_SH     2
+#define HAWAII_RB_BITMAP_WIDTH_PER_SH  4
 
 /* DIDT IND registers */
 #define DIDT_SQ_CTRL0                                     0x0
@@ -499,6 +501,7 @@
                  * bit 4: write
                  */
 #define         MEMORY_CLIENT_ID_MASK                   (0xff << 12)
+#define         HAWAII_MEMORY_CLIENT_ID_MASK            (0x1ff << 12)
 #define         MEMORY_CLIENT_ID_SHIFT                  12
 #define         MEMORY_CLIENT_RW_MASK                   (1 << 24)
 #define         MEMORY_CLIENT_RW_SHIFT                  24
@@ -906,6 +909,39 @@
 #define DPG_PIPE_STUTTER_CONTROL                          0x6cd4
 #       define STUTTER_ENABLE                             (1 << 0)
 
+/* DCE8 FMT blocks */
+#define FMT_DYNAMIC_EXP_CNTL                 0x6fb4
+#       define FMT_DYNAMIC_EXP_EN            (1 << 0)
+#       define FMT_DYNAMIC_EXP_MODE          (1 << 4)
+        /* 0 = 10bit -> 12bit, 1 = 8bit -> 12bit */
+#define FMT_CONTROL                          0x6fb8
+#       define FMT_PIXEL_ENCODING            (1 << 16)
+        /* 0 = RGB 4:4:4 or YCbCr 4:4:4, 1 = YCbCr 4:2:2 */
+#define FMT_BIT_DEPTH_CONTROL                0x6fc8
+#       define FMT_TRUNCATE_EN               (1 << 0)
+#       define FMT_TRUNCATE_MODE             (1 << 1)
+#       define FMT_TRUNCATE_DEPTH(x)         ((x) << 4) /* 0 - 18bpp, 1 - 24bpp, 2 - 30bpp */
+#       define FMT_SPATIAL_DITHER_EN         (1 << 8)
+#       define FMT_SPATIAL_DITHER_MODE(x)    ((x) << 9)
+#       define FMT_SPATIAL_DITHER_DEPTH(x)   ((x) << 11) /* 0 - 18bpp, 1 - 24bpp, 2 - 30bpp */
+#       define FMT_FRAME_RANDOM_ENABLE       (1 << 13)
+#       define FMT_RGB_RANDOM_ENABLE         (1 << 14)
+#       define FMT_HIGHPASS_RANDOM_ENABLE    (1 << 15)
+#       define FMT_TEMPORAL_DITHER_EN        (1 << 16)
+#       define FMT_TEMPORAL_DITHER_DEPTH(x)  ((x) << 17) /* 0 - 18bpp, 1 - 24bpp, 2 - 30bpp */
+#       define FMT_TEMPORAL_DITHER_OFFSET(x) ((x) << 21)
+#       define FMT_TEMPORAL_LEVEL            (1 << 24)
+#       define FMT_TEMPORAL_DITHER_RESET     (1 << 25)
+#       define FMT_25FRC_SEL(x)              ((x) << 26)
+#       define FMT_50FRC_SEL(x)              ((x) << 28)
+#       define FMT_75FRC_SEL(x)              ((x) << 30)
+#define FMT_CLAMP_CONTROL                    0x6fe4
+#       define FMT_CLAMP_DATA_EN             (1 << 0)
+#       define FMT_CLAMP_COLOR_FORMAT(x)     ((x) << 16)
+#       define FMT_CLAMP_6BPC                0
+#       define FMT_CLAMP_8BPC                1
+#       define FMT_CLAMP_10BPC               2
+
 #define GRBM_CNTL                                       0x8000
 #define         GRBM_READ_TIMEOUT(x)                            ((x) << 0)
 
@@ -1129,6 +1165,8 @@
 #              define   ADDR_SURF_P8_32x32_16x16                12
 #              define   ADDR_SURF_P8_32x32_16x32                13
 #              define   ADDR_SURF_P8_32x64_32x32                14
+#              define   ADDR_SURF_P16_32x32_8x16                16
+#              define   ADDR_SURF_P16_32x32_16x16               17
 #       define TILE_SPLIT(x)                                    ((x) << 11)
 #              define   ADDR_SURF_TILE_SPLIT_64B                0
 #              define   ADDR_SURF_TILE_SPLIT_128B               1
@@ -1422,6 +1460,7 @@
 #       define RASTER_CONFIG_RB_MAP_1                   1
 #       define RASTER_CONFIG_RB_MAP_2                   2
 #       define RASTER_CONFIG_RB_MAP_3                   3
+#define         PKR_MAP(x)                              ((x) << 8)
 
 #define VGT_EVENT_INITIATOR                             0x28a90
 #       define SAMPLE_STREAMOUTSTATS1                   (1 << 0)
@@ -1714,6 +1753,68 @@
 #              define PACKET3_PREAMBLE_BEGIN_CLEAR_STATE     (2 << 28)
 #              define PACKET3_PREAMBLE_END_CLEAR_STATE       (3 << 28)
 #define PACKET3_DMA_DATA                                0x50
+/* 1. header
+ * 2. CONTROL
+ * 3. SRC_ADDR_LO or DATA [31:0]
+ * 4. SRC_ADDR_HI [31:0]
+ * 5. DST_ADDR_LO [31:0]
+ * 6. DST_ADDR_HI [7:0]
+ * 7. COMMAND [30:21] | BYTE_COUNT [20:0]
+ */
+/* CONTROL */
+#              define PACKET3_DMA_DATA_ENGINE(x)     ((x) << 0)
+                /* 0 - ME
+                 * 1 - PFP
+                 */
+#              define PACKET3_DMA_DATA_SRC_CACHE_POLICY(x) ((x) << 13)
+                /* 0 - LRU
+                 * 1 - Stream
+                 * 2 - Bypass
+                 */
+#              define PACKET3_DMA_DATA_SRC_VOLATILE (1 << 15)
+#              define PACKET3_DMA_DATA_DST_SEL(x)  ((x) << 20)
+                /* 0 - DST_ADDR using DAS
+                 * 1 - GDS
+                 * 3 - DST_ADDR using L2
+                 */
+#              define PACKET3_DMA_DATA_DST_CACHE_POLICY(x) ((x) << 25)
+                /* 0 - LRU
+                 * 1 - Stream
+                 * 2 - Bypass
+                 */
+#              define PACKET3_DMA_DATA_DST_VOLATILE (1 << 27)
+#              define PACKET3_DMA_DATA_SRC_SEL(x)  ((x) << 29)
+                /* 0 - SRC_ADDR using SAS
+                 * 1 - GDS
+                 * 2 - DATA
+                 * 3 - SRC_ADDR using L2
+                 */
+#              define PACKET3_DMA_DATA_CP_SYNC     (1 << 31)
+/* COMMAND */
+#              define PACKET3_DMA_DATA_DIS_WC      (1 << 21)
+#              define PACKET3_DMA_DATA_CMD_SRC_SWAP(x) ((x) << 22)
+                /* 0 - none
+                 * 1 - 8 in 16
+                 * 2 - 8 in 32
+                 * 3 - 8 in 64
+                 */
+#              define PACKET3_DMA_DATA_CMD_DST_SWAP(x) ((x) << 24)
+                /* 0 - none
+                 * 1 - 8 in 16
+                 * 2 - 8 in 32
+                 * 3 - 8 in 64
+                 */
+#              define PACKET3_DMA_DATA_CMD_SAS     (1 << 26)
+                /* 0 - memory
+                 * 1 - register
+                 */
+#              define PACKET3_DMA_DATA_CMD_DAS     (1 << 27)
+                /* 0 - memory
+                 * 1 - register
+                 */
+#              define PACKET3_DMA_DATA_CMD_SAIC    (1 << 28)
+#              define PACKET3_DMA_DATA_CMD_DAIC    (1 << 29)
+#              define PACKET3_DMA_DATA_CMD_RAW_WAIT  (1 << 30)
 #define PACKET3_AQUIRE_MEM                              0x58
 #define PACKET3_REWIND                                  0x59
 #define PACKET3_LOAD_UCONFIG_REG                        0x5E
diff --git a/drivers/gpu/drm/radeon/dce6_afmt.c b/drivers/gpu/drm/radeon/dce6_afmt.c
index 9fcd338c0fcf..009f46e0ce72 100644
--- a/drivers/gpu/drm/radeon/dce6_afmt.c
+++ b/drivers/gpu/drm/radeon/dce6_afmt.c
@@ -102,6 +102,49 @@ void dce6_afmt_select_pin(struct drm_encoder *encoder)
                AFMT_AUDIO_SRC_SELECT(dig->afmt->pin->id));
 }
 
+void dce6_afmt_write_latency_fields(struct drm_encoder *encoder,
+                                    struct drm_display_mode *mode)
+{
+        struct radeon_device *rdev = encoder->dev->dev_private;
+        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+        struct radeon_encoder_atom_dig *dig = radeon_encoder->enc_priv;
+        struct drm_connector *connector;
+        struct radeon_connector *radeon_connector = NULL;
+        u32 tmp = 0, offset;
+
+        if (!dig->afmt->pin)
+                return;
+
+        offset = dig->afmt->pin->offset;
+
+        list_for_each_entry(connector, &encoder->dev->mode_config.connector_list, head) {
+                if (connector->encoder == encoder) {
+                        radeon_connector = to_radeon_connector(connector);
+                        break;
+                }
+        }
+
+        if (!radeon_connector) {
+                DRM_ERROR("Couldn't find encoder's connector\n");
+                return;
+        }
+
+        if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
+                if (connector->latency_present[1])
+                        tmp = VIDEO_LIPSYNC(connector->video_latency[1]) |
+                                AUDIO_LIPSYNC(connector->audio_latency[1]);
+                else
+                        tmp = VIDEO_LIPSYNC(255) | AUDIO_LIPSYNC(255);
+        } else {
+                if (connector->latency_present[0])
+                        tmp = VIDEO_LIPSYNC(connector->video_latency[0]) |
+                                AUDIO_LIPSYNC(connector->audio_latency[0]);
+                else
+                        tmp = VIDEO_LIPSYNC(255) | AUDIO_LIPSYNC(255);
+        }
+        WREG32_ENDPOINT(offset, AZ_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC, tmp);
+}
+
 void dce6_afmt_write_speaker_allocation(struct drm_encoder *encoder)
 {
         struct radeon_device *rdev = encoder->dev->dev_private;
@@ -113,9 +156,6 @@ void dce6_afmt_write_speaker_allocation(struct drm_encoder *encoder)
         u8 *sadb;
         int sad_count;
 
-        /* XXX: setting this register causes hangs on some asics */
-        return;
-
         if (!dig->afmt->pin)
                 return;
 
@@ -201,20 +241,30 @@ void dce6_afmt_write_sad_regs(struct drm_encoder *encoder)
 
         for (i = 0; i < ARRAY_SIZE(eld_reg_to_type); i++) {
                 u32 value = 0;
+                u8 stereo_freqs = 0;
+                int max_channels = -1;
                 int j;
 
                 for (j = 0; j < sad_count; j++) {
                         struct cea_sad *sad = &sads[j];
 
                         if (sad->format == eld_reg_to_type[i][1]) {
-                                value = MAX_CHANNELS(sad->channels) |
-                                        DESCRIPTOR_BYTE_2(sad->byte2) |
-                                        SUPPORTED_FREQUENCIES(sad->freq);
+                                if (sad->channels > max_channels) {
+                                        value = MAX_CHANNELS(sad->channels) |
+                                                DESCRIPTOR_BYTE_2(sad->byte2) |
+                                                SUPPORTED_FREQUENCIES(sad->freq);
+                                        max_channels = sad->channels;
+                                }
+
                                 if (sad->format == HDMI_AUDIO_CODING_TYPE_PCM)
-                                        value |= SUPPORTED_FREQUENCIES_STEREO(sad->freq);
-                                break;
+                                        stereo_freqs |= sad->freq;
+                                else
+                                        break;
                         }
                 }
+
+                value |= SUPPORTED_FREQUENCIES_STEREO(stereo_freqs);
+
                 WREG32_ENDPOINT(offset, eld_reg_to_type[i][0], value);
         }
 
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index b5c67a99dda9..52f1ae16f653 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -1193,6 +1193,62 @@ void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
         }
 }
 
+void dce4_program_fmt(struct drm_encoder *encoder)
+{
+        struct drm_device *dev = encoder->dev;
+        struct radeon_device *rdev = dev->dev_private;
+        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+        struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
+        struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
+        int bpc = 0;
+        u32 tmp = 0;
+        enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
+
+        if (connector) {
+                struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+                bpc = radeon_get_monitor_bpc(connector);
+                dither = radeon_connector->dither;
+        }
+
+        /* LVDS/eDP FMT is set up by atom */
+        if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
+                return;
+
+        /* not needed for analog */
+        if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
+            (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
+                return;
+
+        if (bpc == 0)
+                return;
+
+        switch (bpc) {
+        case 6:
+                if (dither == RADEON_FMT_DITHER_ENABLE)
+                        /* XXX sort out optimal dither settings */
+                        tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
+                                FMT_SPATIAL_DITHER_EN);
+                else
+                        tmp |= FMT_TRUNCATE_EN;
+                break;
+        case 8:
+                if (dither == RADEON_FMT_DITHER_ENABLE)
+                        /* XXX sort out optimal dither settings */
+                        tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
+                                FMT_RGB_RANDOM_ENABLE |
+                                FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
+                else
+                        tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
+                break;
+        case 10:
+        default:
+                /* not needed */
+                break;
+        }
+
+        WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
+}
+
 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
 {
         if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
@@ -3963,7 +4019,7 @@ int sumo_rlc_init(struct radeon_device *rdev)
                 if (rdev->family >= CHIP_TAHITI) {
                         /* SI */
                         for (i = 0; i < rdev->rlc.reg_list_size; i++)
-                                dst_ptr[i] = src_ptr[i];
+                                dst_ptr[i] = cpu_to_le32(src_ptr[i]);
                 } else {
                         /* ON/LN/TN */
                         /* format:
@@ -3977,10 +4033,10 @@ int sumo_rlc_init(struct radeon_device *rdev)
                                 if (i < dws)
                                         data |= (src_ptr[i] >> 2) << 16;
                                 j = (((i - 1) * 3) / 2);
-                                dst_ptr[j] = data;
+                                dst_ptr[j] = cpu_to_le32(data);
                         }
                         j = ((i * 3) / 2);
-                        dst_ptr[j] = RLC_SAVE_RESTORE_LIST_END_MARKER;
+                        dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
                 }
                 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
@@ -4042,40 +4098,40 @@ int sumo_rlc_init(struct radeon_device *rdev)
                         cik_get_csb_buffer(rdev, dst_ptr);
                 } else if (rdev->family >= CHIP_TAHITI) {
                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
-                        dst_ptr[0] = upper_32_bits(reg_list_mc_addr);
-                        dst_ptr[1] = lower_32_bits(reg_list_mc_addr);
-                        dst_ptr[2] = rdev->rlc.clear_state_size;
+                        dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
+                        dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
+                        dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
                         si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
                 } else {
                         reg_list_hdr_blk_index = 0;
                         reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
                         data = upper_32_bits(reg_list_mc_addr);
-                        dst_ptr[reg_list_hdr_blk_index] = data;
+                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
                         reg_list_hdr_blk_index++;
                         for (i = 0; cs_data[i].section != NULL; i++) {
                                 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
                                         reg_num = cs_data[i].section[j].reg_count;
                                         data = reg_list_mc_addr & 0xffffffff;
-                                        dst_ptr[reg_list_hdr_blk_index] = data;
+                                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
                                         reg_list_hdr_blk_index++;
 
                                         data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
-                                        dst_ptr[reg_list_hdr_blk_index] = data;
+                                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
                                         reg_list_hdr_blk_index++;
 
                                         data = 0x08000000 | (reg_num * 4);
-                                        dst_ptr[reg_list_hdr_blk_index] = data;
+                                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
                                         reg_list_hdr_blk_index++;
 
                                         for (k = 0; k < reg_num; k++) {
                                                 data = cs_data[i].section[j].extent[k];
-                                                dst_ptr[reg_list_blk_index + k] = data;
+                                                dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
                                         }
                                         reg_list_mc_addr += reg_num * 4;
                                         reg_list_blk_index += reg_num;
                                 }
                         }
-                        dst_ptr[reg_list_hdr_blk_index] = RLC_CLEAR_STATE_END_MARKER;
+                        dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
                 }
                 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
diff --git a/drivers/gpu/drm/radeon/evergreen_hdmi.c b/drivers/gpu/drm/radeon/evergreen_hdmi.c
index 57fcc4b16a52..aa695c4feb3d 100644
--- a/drivers/gpu/drm/radeon/evergreen_hdmi.c
+++ b/drivers/gpu/drm/radeon/evergreen_hdmi.c
@@ -35,6 +35,8 @@
 extern void dce6_afmt_write_speaker_allocation(struct drm_encoder *encoder);
 extern void dce6_afmt_write_sad_regs(struct drm_encoder *encoder);
 extern void dce6_afmt_select_pin(struct drm_encoder *encoder);
+extern void dce6_afmt_write_latency_fields(struct drm_encoder *encoder,
+                                           struct drm_display_mode *mode);
 
 /*
  * update the N and CTS parameters for a given pixel clock rate
@@ -58,6 +60,42 @@ static void evergreen_hdmi_update_ACR(struct drm_encoder *encoder, uint32_t cloc
         WREG32(HDMI_ACR_48_1 + offset, acr.n_48khz);
 }
 
+static void dce4_afmt_write_latency_fields(struct drm_encoder *encoder,
+                                           struct drm_display_mode *mode)
+{
+        struct radeon_device *rdev = encoder->dev->dev_private;
+        struct drm_connector *connector;
+        struct radeon_connector *radeon_connector = NULL;
+        u32 tmp = 0;
+
+        list_for_each_entry(connector, &encoder->dev->mode_config.connector_list, head) {
+                if (connector->encoder == encoder) {
+                        radeon_connector = to_radeon_connector(connector);
+                        break;
+                }
+        }
+
+        if (!radeon_connector) {
+                DRM_ERROR("Couldn't find encoder's connector\n");
+                return;
+        }
+
+        if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
+                if (connector->latency_present[1])
+                        tmp = VIDEO_LIPSYNC(connector->video_latency[1]) |
+                                AUDIO_LIPSYNC(connector->audio_latency[1]);
+                else
+                        tmp = VIDEO_LIPSYNC(255) | AUDIO_LIPSYNC(255);
+        } else {
+                if (connector->latency_present[0])
+                        tmp = VIDEO_LIPSYNC(connector->video_latency[0]) |
+                                AUDIO_LIPSYNC(connector->audio_latency[0]);
+                else
+                        tmp = VIDEO_LIPSYNC(255) | AUDIO_LIPSYNC(255);
+        }
+        WREG32(AZ_F0_CODEC_PIN0_CONTROL_RESPONSE_LIPSYNC, tmp);
+}
+
 static void dce4_afmt_write_speaker_allocation(struct drm_encoder *encoder)
 {
         struct radeon_device *rdev = encoder->dev->dev_private;
@@ -67,12 +105,11 @@ static void dce4_afmt_write_speaker_allocation(struct drm_encoder *encoder)
         u8 *sadb;
         int sad_count;
 
-        /* XXX: setting this register causes hangs on some asics */
-        return;
-
         list_for_each_entry(connector, &encoder->dev->mode_config.connector_list, head) {
-                if (connector->encoder == encoder)
+                if (connector->encoder == encoder) {
                         radeon_connector = to_radeon_connector(connector);
+                        break;
+                }
         }
 
         if (!radeon_connector) {
@@ -124,8 +161,10 @@ static void evergreen_hdmi_write_sad_regs(struct drm_encoder *encoder)
         };
 
         list_for_each_entry(connector, &encoder->dev->mode_config.connector_list, head) {
-                if (connector->encoder == encoder)
+                if (connector->encoder == encoder) {
                         radeon_connector = to_radeon_connector(connector);
+                        break;
+                }
         }
 
         if (!radeon_connector) {
@@ -142,20 +181,30 @@ static void evergreen_hdmi_write_sad_regs(struct drm_encoder *encoder)
 
         for (i = 0; i < ARRAY_SIZE(eld_reg_to_type); i++) {
                 u32 value = 0;
+                u8 stereo_freqs = 0;
+                int max_channels = -1;
                 int j;
 
                 for (j = 0; j < sad_count; j++) {
                         struct cea_sad *sad = &sads[j];
 
                         if (sad->format == eld_reg_to_type[i][1]) {
-                                value = MAX_CHANNELS(sad->channels) |
-                                        DESCRIPTOR_BYTE_2(sad->byte2) |
-                                        SUPPORTED_FREQUENCIES(sad->freq);
+                                if (sad->channels > max_channels) {
+                                        value = MAX_CHANNELS(sad->channels) |
+                                                DESCRIPTOR_BYTE_2(sad->byte2) |
+                                                SUPPORTED_FREQUENCIES(sad->freq);
+                                        max_channels = sad->channels;
+                                }
+
                                 if (sad->format == HDMI_AUDIO_CODING_TYPE_PCM)
-                                        value |= SUPPORTED_FREQUENCIES_STEREO(sad->freq);
-                                break;
+                                        stereo_freqs |= sad->freq;
+                                else
+                                        break;
                         }
                 }
+
+                value |= SUPPORTED_FREQUENCIES_STEREO(stereo_freqs);
+
                 WREG32(eld_reg_to_type[i][0], value);
         }
 
@@ -324,8 +373,10 @@ void evergreen_hdmi_setmode(struct drm_encoder *encoder, struct drm_display_mode
         if (ASIC_IS_DCE6(rdev)) {
                 dce6_afmt_select_pin(encoder);
                 dce6_afmt_write_sad_regs(encoder);
+                dce6_afmt_write_latency_fields(encoder, mode);
         } else {
                 evergreen_hdmi_write_sad_regs(encoder);
+                dce4_afmt_write_latency_fields(encoder, mode);
         }
 
         err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode);
diff --git a/drivers/gpu/drm/radeon/evergreend.h b/drivers/gpu/drm/radeon/evergreend.h
index 4f6d2962767d..17f990798992 100644
--- a/drivers/gpu/drm/radeon/evergreend.h
+++ b/drivers/gpu/drm/radeon/evergreend.h
@@ -750,6 +750,44 @@
  * bit6 = 192 kHz
  */
 
+#define AZ_CHANNEL_COUNT_CONTROL                          0x5fe4
+#       define HBR_CHANNEL_COUNT(x)                       (((x) & 0x7) << 0)
+#       define COMPRESSED_CHANNEL_COUNT(x)                (((x) & 0x7) << 4)
+/* HBR_CHANNEL_COUNT, COMPRESSED_CHANNEL_COUNT
+ * 0   = use stream header
+ * 1-7 = channel count - 1
+ */
+#define AZ_F0_CODEC_PIN0_CONTROL_RESPONSE_LIPSYNC         0x5fe8
+#       define VIDEO_LIPSYNC(x)                           (((x) & 0xff) << 0)
+#       define AUDIO_LIPSYNC(x)                           (((x) & 0xff) << 8)
+/* VIDEO_LIPSYNC, AUDIO_LIPSYNC
+ * 0   = invalid
+ * x   = legal delay value
+ * 255 = sync not supported
+ */
+#define AZ_F0_CODEC_PIN0_CONTROL_RESPONSE_HBR             0x5fec
+#       define HBR_CAPABLE                                (1 << 0) /* enabled by default */
+
+#define AZ_F0_CODEC_PIN0_CONTROL_RESPONSE_AV_ASSOCIATION0 0x5ff4
+#       define DISPLAY0_TYPE(x)                           (((x) & 0x3) << 0)
+#       define DISPLAY_TYPE_NONE                   0
+#       define DISPLAY_TYPE_HDMI                   1
+#       define DISPLAY_TYPE_DP                     2
+#       define DISPLAY0_ID(x)                             (((x) & 0x3f) << 2)
+#       define DISPLAY1_TYPE(x)                           (((x) & 0x3) << 8)
+#       define DISPLAY1_ID(x)                             (((x) & 0x3f) << 10)
+#       define DISPLAY2_TYPE(x)                           (((x) & 0x3) << 16)
+#       define DISPLAY2_ID(x)                             (((x) & 0x3f) << 18)
+#       define DISPLAY3_TYPE(x)                           (((x) & 0x3) << 24)
+#       define DISPLAY3_ID(x)                             (((x) & 0x3f) << 26)
+#define AZ_F0_CODEC_PIN0_CONTROL_RESPONSE_AV_ASSOCIATION1 0x5ff8
+#       define DISPLAY4_TYPE(x)                           (((x) & 0x3) << 0)
+#       define DISPLAY4_ID(x)                             (((x) & 0x3f) << 2)
+#       define DISPLAY5_TYPE(x)                           (((x) & 0x3) << 8)
+#       define DISPLAY5_ID(x)                             (((x) & 0x3f) << 10)
+#define AZ_F0_CODEC_PIN0_CONTROL_RESPONSE_AV_NUMBER       0x5ffc
+#       define NUMBER_OF_DISPLAY_ID(x)                    (((x) & 0x7) << 0)
+
 #define AZ_HOT_PLUG_CONTROL                               0x5e78
 #       define AZ_FORCE_CODEC_WAKE                        (1 << 0)
 #       define PIN0_JACK_DETECTION_ENABLE                 (1 << 4)
@@ -1312,6 +1350,38 @@
 #       define DC_HPDx_RX_INT_TIMER(x)                    ((x) << 16)
 #       define DC_HPDx_EN                                 (1 << 28)
 
+/* DCE4/5/6 FMT blocks */
+#define FMT_DYNAMIC_EXP_CNTL                 0x6fb4
+#       define FMT_DYNAMIC_EXP_EN            (1 << 0)
+#       define FMT_DYNAMIC_EXP_MODE          (1 << 4)
+        /* 0 = 10bit -> 12bit, 1 = 8bit -> 12bit */
+#define FMT_CONTROL                          0x6fb8
+#       define FMT_PIXEL_ENCODING            (1 << 16)
+        /* 0 = RGB 4:4:4 or YCbCr 4:4:4, 1 = YCbCr 4:2:2 */
+#define FMT_BIT_DEPTH_CONTROL                0x6fc8
+#       define FMT_TRUNCATE_EN               (1 << 0)
+#       define FMT_TRUNCATE_DEPTH            (1 << 4)
+#       define FMT_SPATIAL_DITHER_EN         (1 << 8)
+#       define FMT_SPATIAL_DITHER_MODE(x)    ((x) << 9)
+#       define FMT_SPATIAL_DITHER_DEPTH      (1 << 12)
+#       define FMT_FRAME_RANDOM_ENABLE       (1 << 13)
+#       define FMT_RGB_RANDOM_ENABLE         (1 << 14)
+#       define FMT_HIGHPASS_RANDOM_ENABLE    (1 << 15)
+#       define FMT_TEMPORAL_DITHER_EN        (1 << 16)
+#       define FMT_TEMPORAL_DITHER_DEPTH     (1 << 20)
+#       define FMT_TEMPORAL_DITHER_OFFSET(x) ((x) << 21)
+#       define FMT_TEMPORAL_LEVEL            (1 << 24)
+#       define FMT_TEMPORAL_DITHER_RESET     (1 << 25)
+#       define FMT_25FRC_SEL(x)              ((x) << 26)
+#       define FMT_50FRC_SEL(x)              ((x) << 28)
+#       define FMT_75FRC_SEL(x)              ((x) << 30)
+#define FMT_CLAMP_CONTROL                    0x6fe4
+#       define FMT_CLAMP_DATA_EN             (1 << 0)
+#       define FMT_CLAMP_COLOR_FORMAT(x)     ((x) << 16)
+#       define FMT_CLAMP_6BPC                0
+#       define FMT_CLAMP_8BPC                1
+#       define FMT_CLAMP_10BPC               2
+
 /* ASYNC DMA */
 #define DMA_RB_RPTR                                       0xd008
 #define DMA_RB_WPTR                                       0xd00c
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
index cac2866d79da..11aab2ab54ce 100644
--- a/drivers/gpu/drm/radeon/ni.c
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -174,11 +174,6 @@ extern void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
 extern void evergreen_program_aspm(struct radeon_device *rdev);
 extern void sumo_rlc_fini(struct radeon_device *rdev);
 extern int sumo_rlc_init(struct radeon_device *rdev);
-extern void cayman_dma_vm_set_page(struct radeon_device *rdev,
-                                   struct radeon_ib *ib,
-                                   uint64_t pe,
-                                   uint64_t addr, unsigned count,
-                                   uint32_t incr, uint32_t flags);
 
 /* Firmware Names */
 MODULE_FIRMWARE("radeon/BARTS_pfp.bin");
@@ -2400,77 +2395,6 @@ void cayman_vm_decode_fault(struct radeon_device *rdev,
                block, mc_id);
 }
 
-#define R600_ENTRY_VALID   (1 << 0)
-#define R600_PTE_SYSTEM    (1 << 1)
-#define R600_PTE_SNOOPED   (1 << 2)
-#define R600_PTE_READABLE  (1 << 5)
-#define R600_PTE_WRITEABLE (1 << 6)
-
-uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags)
-{
-        uint32_t r600_flags = 0;
-        r600_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_ENTRY_VALID : 0;
-        r600_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0;
-        r600_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0;
-        if (flags & RADEON_VM_PAGE_SYSTEM) {
-                r600_flags |= R600_PTE_SYSTEM;
-                r600_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0;
-        }
-        return r600_flags;
-}
-
-/**
- * cayman_vm_set_page - update the page tables using the CP
- *
- * @rdev: radeon_device pointer
- * @ib: indirect buffer to fill with commands
- * @pe: addr of the page entry
- * @addr: dst addr to write into pe
- * @count: number of page entries to update
- * @incr: increase next addr by incr bytes
- * @flags: access flags
- *
- * Update the page tables using the CP (cayman/TN).
- */
-void cayman_vm_set_page(struct radeon_device *rdev,
-                        struct radeon_ib *ib,
-                        uint64_t pe,
-                        uint64_t addr, unsigned count,
-                        uint32_t incr, uint32_t flags)
-{
-        uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
-        uint64_t value;
-        unsigned ndw;
-
-        if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
-                while (count) {
-                        ndw = 1 + count * 2;
-                        if (ndw > 0x3FFF)
-                                ndw = 0x3FFF;
-
-                        ib->ptr[ib->length_dw++] = PACKET3(PACKET3_ME_WRITE, ndw);
-                        ib->ptr[ib->length_dw++] = pe;
-                        ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
-                        for (; ndw > 1; ndw -= 2, --count, pe += 8) {
-                                if (flags & RADEON_VM_PAGE_SYSTEM) {
-                                        value = radeon_vm_map_gart(rdev, addr);
-                                        value &= 0xFFFFFFFFFFFFF000ULL;
-                                } else if (flags & RADEON_VM_PAGE_VALID) {
-                                        value = addr;
-                                } else {
-                                        value = 0;
-                                }
-                                addr += incr;
-                                value |= r600_flags;
-                                ib->ptr[ib->length_dw++] = value;
-                                ib->ptr[ib->length_dw++] = upper_32_bits(value);
-                        }
-                }
-        } else {
-                cayman_dma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
-        }
-}
-
 /**
  * cayman_vm_flush - vm flush using the CP
  *
diff --git a/drivers/gpu/drm/radeon/ni_dma.c b/drivers/gpu/drm/radeon/ni_dma.c
index dd6e9688fbef..bdeb65ed3658 100644
--- a/drivers/gpu/drm/radeon/ni_dma.c
+++ b/drivers/gpu/drm/radeon/ni_dma.c
@@ -24,6 +24,7 @@
 #include <drm/drmP.h>
 #include "radeon.h"
 #include "radeon_asic.h"
+#include "radeon_trace.h"
 #include "nid.h"
 
 u32 cayman_gpu_check_soft_reset(struct radeon_device *rdev);
@@ -245,8 +246,7 @@ bool cayman_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
  * @addr: dst addr to write into pe
  * @count: number of page entries to update
  * @incr: increase next addr by incr bytes
- * @flags: access flags
- * @r600_flags: hw access flags 
+ * @flags: hw access flags 
  *
  * Update the page tables using the DMA (cayman/TN).
  */
@@ -256,11 +256,12 @@ void cayman_dma_vm_set_page(struct radeon_device *rdev,
                             uint64_t addr, unsigned count,
                             uint32_t incr, uint32_t flags)
 {
-        uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
         uint64_t value;
         unsigned ndw;
 
-        if ((flags & RADEON_VM_PAGE_SYSTEM) || (count == 1)) {
+        trace_radeon_vm_set_page(pe, addr, count, incr, flags);
+
+        if ((flags & R600_PTE_SYSTEM) || (count == 1)) {
                 while (count) {
                         ndw = count * 2;
                         if (ndw > 0xFFFFE)
@@ -271,16 +272,16 @@ void cayman_dma_vm_set_page(struct radeon_device *rdev,
                         ib->ptr[ib->length_dw++] = pe;
                         ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
                         for (; ndw > 0; ndw -= 2, --count, pe += 8) {
-                                if (flags & RADEON_VM_PAGE_SYSTEM) {
+                                if (flags & R600_PTE_SYSTEM) {
                                         value = radeon_vm_map_gart(rdev, addr);
                                         value &= 0xFFFFFFFFFFFFF000ULL;
-                                } else if (flags & RADEON_VM_PAGE_VALID) {
+                                } else if (flags & R600_PTE_VALID) {
                                         value = addr;
                                 } else {
                                         value = 0;
                                 }
                                 addr += incr;
-                                value |= r600_flags;
+                                value |= flags;
                                 ib->ptr[ib->length_dw++] = value;
                                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
                         }
@@ -291,7 +292,7 @@ void cayman_dma_vm_set_page(struct radeon_device *rdev,
                         if (ndw > 0xFFFFE)
                                 ndw = 0xFFFFE;
 
-                        if (flags & RADEON_VM_PAGE_VALID)
+                        if (flags & R600_PTE_VALID)
                                 value = addr;
                         else
                                 value = 0;
@@ -299,7 +300,7 @@ void cayman_dma_vm_set_page(struct radeon_device *rdev,
                         ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
                         ib->ptr[ib->length_dw++] = pe; /* dst addr */
                         ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
-                        ib->ptr[ib->length_dw++] = r600_flags; /* mask */
+                        ib->ptr[ib->length_dw++] = flags; /* mask */
                         ib->ptr[ib->length_dw++] = 0;
                         ib->ptr[ib->length_dw++] = value; /* value */
                         ib->ptr[ib->length_dw++] = upper_32_bits(value);
diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index d71333033b2b..784983d78158 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -1434,7 +1434,7 @@ int r100_cs_packet_parse_vline(struct radeon_cs_parser *p)
         obj = drm_mode_object_find(p->rdev->ddev, crtc_id, DRM_MODE_OBJECT_CRTC);
         if (!obj) {
                 DRM_ERROR("cannot find crtc %d\n", crtc_id);
-                return -EINVAL;
+                return -ENOENT;
         }
         crtc = obj_to_crtc(obj);
         radeon_crtc = to_radeon_crtc(crtc);
diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index f9be22062df1..4e609e8a8d2b 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -124,6 +124,59 @@ int r600_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
         return 0;
 }
 
+void dce3_program_fmt(struct drm_encoder *encoder)
+{
+        struct drm_device *dev = encoder->dev;
+        struct radeon_device *rdev = dev->dev_private;
+        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+        struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
+        struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
+        int bpc = 0;
+        u32 tmp = 0;
+        enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
+
+        if (connector) {
+                struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+                bpc = radeon_get_monitor_bpc(connector);
+                dither = radeon_connector->dither;
+        }
+
+        /* LVDS FMT is set up by atom */
+        if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
+                return;
+
+        /* not needed for analog */
+        if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
+            (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
+                return;
+
+        if (bpc == 0)
+                return;
+
+        switch (bpc) {
+        case 6:
+                if (dither == RADEON_FMT_DITHER_ENABLE)
+                        /* XXX sort out optimal dither settings */
+                        tmp |= FMT_SPATIAL_DITHER_EN;
+                else
+                        tmp |= FMT_TRUNCATE_EN;
+                break;
+        case 8:
+                if (dither == RADEON_FMT_DITHER_ENABLE)
+                        /* XXX sort out optimal dither settings */
+                        tmp |= (FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
+                else
+                        tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
+                break;
+        case 10:
+        default:
+                /* not needed */
+                break;
+        }
+
+        WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
+}
+
 /* get temperature in millidegrees */
 int rv6xx_get_temp(struct radeon_device *rdev)
 {
diff --git a/drivers/gpu/drm/radeon/r600_cs.c b/drivers/gpu/drm/radeon/r600_cs.c
index 01a3ec83f284..5dceea6f71ae 100644
--- a/drivers/gpu/drm/radeon/r600_cs.c
+++ b/drivers/gpu/drm/radeon/r600_cs.c
@@ -887,7 +887,7 @@ int r600_cs_common_vline_parse(struct radeon_cs_parser *p,
         obj = drm_mode_object_find(p->rdev->ddev, crtc_id, DRM_MODE_OBJECT_CRTC);
         if (!obj) {
                 DRM_ERROR("cannot find crtc %d\n", crtc_id);
-                return -EINVAL;
+                return -ENOENT;
         }
         crtc = obj_to_crtc(obj);
         radeon_crtc = to_radeon_crtc(crtc);
@@ -2328,13 +2328,8 @@ static void r600_cs_parser_fini(struct radeon_cs_parser *parser, int error)
         unsigned i;
 
         kfree(parser->relocs);
-        for (i = 0; i < parser->nchunks; i++) {
-                kfree(parser->chunks[i].kdata);
-                if (parser->rdev && (parser->rdev->flags & RADEON_IS_AGP)) {
-                        kfree(parser->chunks[i].kpage[0]);
-                        kfree(parser->chunks[i].kpage[1]);
-                }
-        }
+        for (i = 0; i < parser->nchunks; i++)
+                drm_free_large(parser->chunks[i].kdata);
         kfree(parser->chunks);
         kfree(parser->chunks_array);
 }
@@ -2391,13 +2386,12 @@ int r600_cs_legacy(struct drm_device *dev, void *data, struct drm_file *filp,
         ib_chunk = &parser.chunks[parser.chunk_ib_idx];
         parser.ib.length_dw = ib_chunk->length_dw;
         *l = parser.ib.length_dw;
-        r = r600_cs_parse(&parser);
-        if (r) {
-                DRM_ERROR("Invalid command stream !\n");
+        if (DRM_COPY_FROM_USER(ib, ib_chunk->user_ptr, ib_chunk->length_dw * 4)) {
+                r = -EFAULT;
                 r600_cs_parser_fini(&parser, r);
                 return r;
         }
-        r = radeon_cs_finish_pages(&parser);
+        r = r600_cs_parse(&parser);
         if (r) {
                 DRM_ERROR("Invalid command stream !\n");
                 r600_cs_parser_fini(&parser, r);
diff --git a/drivers/gpu/drm/radeon/r600_hdmi.c b/drivers/gpu/drm/radeon/r600_hdmi.c
index 06022e3b9c3b..4b89262f3f0e 100644
--- a/drivers/gpu/drm/radeon/r600_hdmi.c
+++ b/drivers/gpu/drm/radeon/r600_hdmi.c
@@ -24,6 +24,7 @@
  * Authors: Christian König
  */
 #include <linux/hdmi.h>
+#include <linux/gcd.h>
 #include <drm/drmP.h>
 #include <drm/radeon_drm.h>
 #include "radeon.h"
@@ -57,35 +58,57 @@ enum r600_hdmi_iec_status_bits {
 static const struct radeon_hdmi_acr r600_hdmi_predefined_acr[] = {
     /*       32kHz        44.1kHz       48kHz    */
     /* Clock      N     CTS      N     CTS      N     CTS */
-    {  25175,  4576,  28125,  7007,  31250,  6864,  28125 }, /*  25,20/1.001 MHz */
+    {  25175,  4096,  25175, 28224, 125875,  6144,  25175 }, /*  25,20/1.001 MHz */
     {  25200,  4096,  25200,  6272,  28000,  6144,  25200 }, /*  25.20       MHz */
     {  27000,  4096,  27000,  6272,  30000,  6144,  27000 }, /*  27.00       MHz */
     {  27027,  4096,  27027,  6272,  30030,  6144,  27027 }, /*  27.00*1.001 MHz */
     {  54000,  4096,  54000,  6272,  60000,  6144,  54000 }, /*  54.00       MHz */
     {  54054,  4096,  54054,  6272,  60060,  6144,  54054 }, /*  54.00*1.001 MHz */
-    {  74176, 11648, 210937, 17836, 234375, 11648, 140625 }, /*  74.25/1.001 MHz */
+    {  74176,  4096,  74176,  5733,  75335,  6144,  74176 }, /*  74.25/1.001 MHz */
     {  74250,  4096,  74250,  6272,  82500,  6144,  74250 }, /*  74.25       MHz */
-    { 148352, 11648, 421875,  8918, 234375,  5824, 140625 }, /* 148.50/1.001 MHz */
+    { 148352,  4096, 148352,  5733, 150670,  6144, 148352 }, /* 148.50/1.001 MHz */
     { 148500,  4096, 148500,  6272, 165000,  6144, 148500 }, /* 148.50       MHz */
-    {      0,  4096,      0,  6272,      0,  6144,      0 }  /* Other */
 };
 
+
 /*
- * calculate CTS value if it's not found in the table
+ * calculate CTS and N values if they are not found in the table
  */
-static void r600_hdmi_calc_cts(uint32_t clock, int *CTS, int N, int freq)
+static void r600_hdmi_calc_cts(uint32_t clock, int *CTS, int *N, int freq)
 {
-        u64 n;
-        u32 d;
-
-        if (*CTS == 0) {
-                n = (u64)clock * (u64)N * 1000ULL;
-                d = 128 * freq;
-                do_div(n, d);
-                *CTS = n;
-        }
-        DRM_DEBUG("Using ACR timing N=%d CTS=%d for frequency %d\n",
-                  N, *CTS, freq);
+        int n, cts;
+        unsigned long div, mul;
+
+        /* Safe, but overly large values */
+        n = 128 * freq;
+        cts = clock * 1000;
+
+        /* Smallest valid fraction */
+        div = gcd(n, cts);
+
+        n /= div;
+        cts /= div;
+
+        /*
+         * The optimal N is 128*freq/1000. Calculate the closest larger
+         * value that doesn't truncate any bits.
+         */
+        mul = ((128*freq/1000) + (n-1))/n;
+
+        n *= mul;
+        cts *= mul;
+
+        /* Check that we are in spec (not always possible) */
+        if (n < (128*freq/1500))
+                printk(KERN_WARNING "Calculated ACR N value is too small. You may experience audio problems.\n");
+        if (n > (128*freq/300))
+                printk(KERN_WARNING "Calculated ACR N value is too large. You may experience audio problems.\n");
+
+        *N = n;
+        *CTS = cts;
+
+        DRM_DEBUG("Calculated ACR timing N=%d CTS=%d for frequency %d\n",
+                  *N, *CTS, freq);
 }
 
 struct radeon_hdmi_acr r600_hdmi_acr(uint32_t clock)
@@ -93,15 +116,16 @@ struct radeon_hdmi_acr r600_hdmi_acr(uint32_t clock)
         struct radeon_hdmi_acr res;
         u8 i;
 
-        for (i = 0; r600_hdmi_predefined_acr[i].clock != clock &&
-             r600_hdmi_predefined_acr[i].clock != 0; i++)
-                ;
-        res = r600_hdmi_predefined_acr[i];
+        /* Precalculated values for common clocks */
+        for (i = 0; i < ARRAY_SIZE(r600_hdmi_predefined_acr); i++) {
+                if (r600_hdmi_predefined_acr[i].clock == clock)
+                        return r600_hdmi_predefined_acr[i];
+        }
 
-        /* In case some CTS are missing */
-        r600_hdmi_calc_cts(clock, &res.cts_32khz, res.n_32khz, 32000);
-        r600_hdmi_calc_cts(clock, &res.cts_44_1khz, res.n_44_1khz, 44100);
-        r600_hdmi_calc_cts(clock, &res.cts_48khz, res.n_48khz, 48000);
+        /* And odd clocks get manually calculated */
+        r600_hdmi_calc_cts(clock, &res.cts_32khz, &res.n_32khz, 32000);
+        r600_hdmi_calc_cts(clock, &res.cts_44_1khz, &res.n_44_1khz, 44100);
+        r600_hdmi_calc_cts(clock, &res.cts_48khz, &res.n_48khz, 48000);
 
         return res;
 }
@@ -313,8 +337,10 @@ static void dce3_2_afmt_write_speaker_allocation(struct drm_encoder *encoder)
         return;
 
         list_for_each_entry(connector, &encoder->dev->mode_config.connector_list, head) {
-                if (connector->encoder == encoder)
+                if (connector->encoder == encoder) {
                         radeon_connector = to_radeon_connector(connector);
+                        break;
+                }
         }
 
         if (!radeon_connector) {
@@ -366,8 +392,10 @@ static void dce3_2_afmt_write_sad_regs(struct drm_encoder *encoder)
         };
 
         list_for_each_entry(connector, &encoder->dev->mode_config.connector_list, head) {
-                if (connector->encoder == encoder)
+                if (connector->encoder == encoder) {
                         radeon_connector = to_radeon_connector(connector);
+                        break;
+                }
         }
 
         if (!radeon_connector) {
@@ -384,20 +412,30 @@ static void dce3_2_afmt_write_sad_regs(struct drm_encoder *encoder)
 
         for (i = 0; i < ARRAY_SIZE(eld_reg_to_type); i++) {
                 u32 value = 0;
+                u8 stereo_freqs = 0;
+                int max_channels = -1;
                 int j;
 
                 for (j = 0; j < sad_count; j++) {
                         struct cea_sad *sad = &sads[j];
 
                         if (sad->format == eld_reg_to_type[i][1]) {
-                                value = MAX_CHANNELS(sad->channels) |
-                                        DESCRIPTOR_BYTE_2(sad->byte2) |
-                                        SUPPORTED_FREQUENCIES(sad->freq);
+                                if (sad->channels > max_channels) {
+                                        value = MAX_CHANNELS(sad->channels) |
+                                                DESCRIPTOR_BYTE_2(sad->byte2) |
+                                                SUPPORTED_FREQUENCIES(sad->freq);
+                                        max_channels = sad->channels;
+                                }
+
                                 if (sad->format == HDMI_AUDIO_CODING_TYPE_PCM)
-                                        value |= SUPPORTED_FREQUENCIES_STEREO(sad->freq);
-                                break;
+                                        stereo_freqs |= sad->freq;
+                                else
+                                        break;
                         }
                 }
+
+                value |= SUPPORTED_FREQUENCIES_STEREO(stereo_freqs);
+
                 WREG32(eld_reg_to_type[i][0], value);
         }
 
diff --git a/drivers/gpu/drm/radeon/r600d.h b/drivers/gpu/drm/radeon/r600d.h
index 7b3c7b5932c5..ebe38724a976 100644
--- a/drivers/gpu/drm/radeon/r600d.h
+++ b/drivers/gpu/drm/radeon/r600d.h
@@ -1199,6 +1199,34 @@
 #       define AFMT_AZ_FORMAT_WTRIG_ACK      (1 << 29)
 #       define AFMT_AZ_AUDIO_ENABLE_CHG_ACK  (1 << 30)
 
+/* DCE3 FMT blocks */
+#define FMT_CONTROL                          0x6700
+#       define FMT_PIXEL_ENCODING            (1 << 16)
+        /* 0 = RGB 4:4:4 or YCbCr 4:4:4, 1 = YCbCr 4:2:2 */
+#define FMT_BIT_DEPTH_CONTROL                0x6710
+#       define FMT_TRUNCATE_EN               (1 << 0)
+#       define FMT_TRUNCATE_DEPTH            (1 << 4)
+#       define FMT_SPATIAL_DITHER_EN         (1 << 8)
+#       define FMT_SPATIAL_DITHER_MODE(x)    ((x) << 9)
+#       define FMT_SPATIAL_DITHER_DEPTH      (1 << 12)
+#       define FMT_FRAME_RANDOM_ENABLE       (1 << 13)
+#       define FMT_RGB_RANDOM_ENABLE         (1 << 14)
+#       define FMT_HIGHPASS_RANDOM_ENABLE    (1 << 15)
+#       define FMT_TEMPORAL_DITHER_EN        (1 << 16)
+#       define FMT_TEMPORAL_DITHER_DEPTH     (1 << 20)
+#       define FMT_TEMPORAL_DITHER_OFFSET(x) ((x) << 21)
+#       define FMT_TEMPORAL_LEVEL            (1 << 24)
+#       define FMT_TEMPORAL_DITHER_RESET     (1 << 25)
+#       define FMT_25FRC_SEL(x)              ((x) << 26)
+#       define FMT_50FRC_SEL(x)              ((x) << 28)
+#       define FMT_75FRC_SEL(x)              ((x) << 30)
+#define FMT_CLAMP_CONTROL                    0x672c
+#       define FMT_CLAMP_DATA_EN             (1 << 0)
+#       define FMT_CLAMP_COLOR_FORMAT(x)     ((x) << 16)
+#       define FMT_CLAMP_6BPC                0
+#       define FMT_CLAMP_8BPC                1
+#       define FMT_CLAMP_10BPC               2
+
 /* Power management */
 #define CG_SPLL_FUNC_CNTL                                 0x600
 #       define SPLL_RESET                                (1 << 0)
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 24f4960f59ee..b9ee99258602 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -98,6 +98,7 @@ extern int radeon_lockup_timeout;
 extern int radeon_fastfb;
 extern int radeon_dpm;
 extern int radeon_aspm;
+extern int radeon_runtime_pm;
 
 /*
  * Copy from radeon_drv.h so we don't have to include both and have conflicting
@@ -327,7 +328,6 @@ struct radeon_fence_driver {
         /* sync_seq is protected by ring emission lock */
         uint64_t                        sync_seq[RADEON_NUM_RINGS];
         atomic64_t                      last_seq;
-        unsigned long                   last_activity;
         bool                            initialized;
 };
 
@@ -832,6 +832,12 @@ struct radeon_mec {
 #define RADEON_VM_PTB_ALIGN_MASK (RADEON_VM_PTB_ALIGN_SIZE - 1)
 #define RADEON_VM_PTB_ALIGN(a) (((a) + RADEON_VM_PTB_ALIGN_MASK) & ~RADEON_VM_PTB_ALIGN_MASK)
 
+#define R600_PTE_VALID          (1 << 0)
+#define R600_PTE_SYSTEM         (1 << 1)
+#define R600_PTE_SNOOPED        (1 << 2)
+#define R600_PTE_READABLE       (1 << 5)
+#define R600_PTE_WRITEABLE      (1 << 6)
+
 struct radeon_vm {
         struct list_head                list;
         struct list_head                va;
@@ -967,12 +973,8 @@ struct radeon_cs_reloc {
 struct radeon_cs_chunk {
         uint32_t                chunk_id;
         uint32_t                length_dw;
-        int                     kpage_idx[2];
-        uint32_t                *kpage[2];
         uint32_t                *kdata;
         void __user             *user_ptr;
-        int                     last_copied_page;
-        int                     last_page_index;
 };
 
 struct radeon_cs_parser {
@@ -1007,8 +1009,15 @@ struct radeon_cs_parser {
         struct ww_acquire_ctx   ticket;
 };
 
-extern int radeon_cs_finish_pages(struct radeon_cs_parser *p);
-extern u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx);
+static inline u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx)
+{
+        struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
+
+        if (ibc->kdata)
+                return ibc->kdata[idx];
+        return p->ib.ptr[idx];
+}
+
 
 struct radeon_cs_packet {
         unsigned        idx;
@@ -1675,8 +1684,6 @@ struct radeon_asic {
         struct {
                 int (*init)(struct radeon_device *rdev);
                 void (*fini)(struct radeon_device *rdev);
-
-                u32 pt_ring_index;
                 void (*set_page)(struct radeon_device *rdev,
                                  struct radeon_ib *ib,
                                  uint64_t pe,
@@ -2170,6 +2177,7 @@ struct radeon_device {
         bool                            need_dma32;
         bool                            accel_working;
         bool                            fastfb_working; /* IGP feature*/
+        bool                            needs_reset;
         struct radeon_surface_reg surface_regs[RADEON_GEM_MAX_SURFACES];
         const struct firmware *me_fw;   /* all family ME firmware */
         const struct firmware *pfp_fw;  /* r6/700 PFP firmware */
@@ -2212,6 +2220,9 @@ struct radeon_device {
         /* clock, powergating flags */
         u32 cg_flags;
         u32 pg_flags;
+
+        struct dev_pm_domain vga_pm_domain;
+        bool have_disp_power_ref;
 };
 
 int radeon_device_init(struct radeon_device *rdev,
@@ -2673,8 +2684,8 @@ extern void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain);
 extern bool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo);
 extern void radeon_vram_location(struct radeon_device *rdev, struct radeon_mc *mc, u64 base);
 extern void radeon_gtt_location(struct radeon_device *rdev, struct radeon_mc *mc);
-extern int radeon_resume_kms(struct drm_device *dev);
-extern int radeon_suspend_kms(struct drm_device *dev, pm_message_t state);
+extern int radeon_resume_kms(struct drm_device *dev, bool resume, bool fbcon);
+extern int radeon_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon);
 extern void radeon_ttm_set_active_vram_size(struct radeon_device *rdev, u64 size);
 extern void radeon_program_register_sequence(struct radeon_device *rdev,
                                              const u32 *registers,
diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
index 8f7e04538fd6..50853c0cb49d 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.c
+++ b/drivers/gpu/drm/radeon/radeon_asic.c
@@ -1622,8 +1622,7 @@ static struct radeon_asic cayman_asic = {
         .vm = {
                 .init = &cayman_vm_init,
                 .fini = &cayman_vm_fini,
-                .pt_ring_index = R600_RING_TYPE_DMA_INDEX,
-                .set_page = &cayman_vm_set_page,
+                .set_page = &cayman_dma_vm_set_page,
         },
         .ring = {
                 [RADEON_RING_TYPE_GFX_INDEX] = &cayman_gfx_ring,
@@ -1723,8 +1722,7 @@ static struct radeon_asic trinity_asic = {
         .vm = {
                 .init = &cayman_vm_init,
                 .fini = &cayman_vm_fini,
-                .pt_ring_index = R600_RING_TYPE_DMA_INDEX,
-                .set_page = &cayman_vm_set_page,
+                .set_page = &cayman_dma_vm_set_page,
         },
         .ring = {
                 [RADEON_RING_TYPE_GFX_INDEX] = &cayman_gfx_ring,
@@ -1854,8 +1852,7 @@ static struct radeon_asic si_asic = {
         .vm = {
                 .init = &si_vm_init,
                 .fini = &si_vm_fini,
-                .pt_ring_index = R600_RING_TYPE_DMA_INDEX,
-                .set_page = &si_vm_set_page,
+                .set_page = &si_dma_vm_set_page,
         },
         .ring = {
                 [RADEON_RING_TYPE_GFX_INDEX] = &si_gfx_ring,
@@ -1879,7 +1876,7 @@ static struct radeon_asic si_asic = {
                 .hdmi_setmode = &evergreen_hdmi_setmode,
         },
         .copy = {
-                .blit = NULL,
+                .blit = &r600_copy_cpdma,
                 .blit_ring_index = RADEON_RING_TYPE_GFX_INDEX,
                 .dma = &si_copy_dma,
                 .dma_ring_index = R600_RING_TYPE_DMA_INDEX,
@@ -2000,8 +1997,7 @@ static struct radeon_asic ci_asic = {
         .vm = {
                 .init = &cik_vm_init,
                 .fini = &cik_vm_fini,
-                .pt_ring_index = R600_RING_TYPE_DMA_INDEX,
-                .set_page = &cik_vm_set_page,
+                .set_page = &cik_sdma_vm_set_page,
         },
         .ring = {
                 [RADEON_RING_TYPE_GFX_INDEX] = &ci_gfx_ring,
@@ -2100,8 +2096,7 @@ static struct radeon_asic kv_asic = {
         .vm = {
                 .init = &cik_vm_init,
                 .fini = &cik_vm_fini,
-                .pt_ring_index = R600_RING_TYPE_DMA_INDEX,
-                .set_page = &cik_vm_set_page,
+                .set_page = &cik_sdma_vm_set_page,
         },
         .ring = {
                 [RADEON_RING_TYPE_GFX_INDEX] = &ci_gfx_ring,
@@ -2442,27 +2437,48 @@ int radeon_asic_init(struct radeon_device *rdev)
                 }
                 break;
         case CHIP_BONAIRE:
+        case CHIP_HAWAII:
                 rdev->asic = &ci_asic;
                 rdev->num_crtc = 6;
                 rdev->has_uvd = true;
-                rdev->cg_flags =
-                        RADEON_CG_SUPPORT_GFX_MGCG |
-                        RADEON_CG_SUPPORT_GFX_MGLS |
-                        /*RADEON_CG_SUPPORT_GFX_CGCG |*/
-                        RADEON_CG_SUPPORT_GFX_CGLS |
-                        RADEON_CG_SUPPORT_GFX_CGTS |
-                        RADEON_CG_SUPPORT_GFX_CGTS_LS |
-                        RADEON_CG_SUPPORT_GFX_CP_LS |
-                        RADEON_CG_SUPPORT_MC_LS |
-                        RADEON_CG_SUPPORT_MC_MGCG |
-                        RADEON_CG_SUPPORT_SDMA_MGCG |
-                        RADEON_CG_SUPPORT_SDMA_LS |
-                        RADEON_CG_SUPPORT_BIF_LS |
-                        RADEON_CG_SUPPORT_VCE_MGCG |
-                        RADEON_CG_SUPPORT_UVD_MGCG |
-                        RADEON_CG_SUPPORT_HDP_LS |
-                        RADEON_CG_SUPPORT_HDP_MGCG;
-                rdev->pg_flags = 0;
+                if (rdev->family == CHIP_BONAIRE) {
+                        rdev->cg_flags =
+                                RADEON_CG_SUPPORT_GFX_MGCG |
+                                RADEON_CG_SUPPORT_GFX_MGLS |
+                                /*RADEON_CG_SUPPORT_GFX_CGCG |*/
+                                RADEON_CG_SUPPORT_GFX_CGLS |
+                                RADEON_CG_SUPPORT_GFX_CGTS |
+                                RADEON_CG_SUPPORT_GFX_CGTS_LS |
+                                RADEON_CG_SUPPORT_GFX_CP_LS |
+                                RADEON_CG_SUPPORT_MC_LS |
+                                RADEON_CG_SUPPORT_MC_MGCG |
+                                RADEON_CG_SUPPORT_SDMA_MGCG |
+                                RADEON_CG_SUPPORT_SDMA_LS |
+                                RADEON_CG_SUPPORT_BIF_LS |
+                                RADEON_CG_SUPPORT_VCE_MGCG |
+                                RADEON_CG_SUPPORT_UVD_MGCG |
+                                RADEON_CG_SUPPORT_HDP_LS |
+                                RADEON_CG_SUPPORT_HDP_MGCG;
+                        rdev->pg_flags = 0;
+                } else {
+                        rdev->cg_flags =
+                                RADEON_CG_SUPPORT_GFX_MGCG |
+                                RADEON_CG_SUPPORT_GFX_MGLS |
+                                /*RADEON_CG_SUPPORT_GFX_CGCG |*/
+                                RADEON_CG_SUPPORT_GFX_CGLS |
+                                RADEON_CG_SUPPORT_GFX_CGTS |
+                                RADEON_CG_SUPPORT_GFX_CP_LS |
+                                RADEON_CG_SUPPORT_MC_LS |
+                                RADEON_CG_SUPPORT_MC_MGCG |
+                                RADEON_CG_SUPPORT_SDMA_MGCG |
+                                RADEON_CG_SUPPORT_SDMA_LS |
+                                RADEON_CG_SUPPORT_BIF_LS |
+                                RADEON_CG_SUPPORT_VCE_MGCG |
+                                RADEON_CG_SUPPORT_UVD_MGCG |
+                                RADEON_CG_SUPPORT_HDP_LS |
+                                RADEON_CG_SUPPORT_HDP_MGCG;
+                        rdev->pg_flags = 0;
+                }
                 break;
         case CHIP_KAVERI:
         case CHIP_KABINI:
diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h
index 70c29d5e080d..f2833ee3a613 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.h
+++ b/drivers/gpu/drm/radeon/radeon_asic.h
@@ -581,17 +581,18 @@ int cayman_vm_init(struct radeon_device *rdev);
 void cayman_vm_fini(struct radeon_device *rdev);
 void cayman_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
 uint32_t cayman_vm_page_flags(struct radeon_device *rdev, uint32_t flags);
-void cayman_vm_set_page(struct radeon_device *rdev,
-                        struct radeon_ib *ib,
-                        uint64_t pe,
-                        uint64_t addr, unsigned count,
-                        uint32_t incr, uint32_t flags);
 int evergreen_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib);
 int evergreen_dma_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib);
 void cayman_dma_ring_ib_execute(struct radeon_device *rdev,
                                 struct radeon_ib *ib);
 bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring);
 bool cayman_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring);
+void cayman_dma_vm_set_page(struct radeon_device *rdev,
+                            struct radeon_ib *ib,
+                            uint64_t pe,
+                            uint64_t addr, unsigned count,
+                            uint32_t incr, uint32_t flags);
+
 void cayman_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
 
 int ni_dpm_init(struct radeon_device *rdev);
@@ -653,17 +654,17 @@ int si_irq_set(struct radeon_device *rdev);
 int si_irq_process(struct radeon_device *rdev);
 int si_vm_init(struct radeon_device *rdev);
 void si_vm_fini(struct radeon_device *rdev);
-void si_vm_set_page(struct radeon_device *rdev,
-                    struct radeon_ib *ib,
-                    uint64_t pe,
-                    uint64_t addr, unsigned count,
-                    uint32_t incr, uint32_t flags);
 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib);
 int si_copy_dma(struct radeon_device *rdev,
                 uint64_t src_offset, uint64_t dst_offset,
                 unsigned num_gpu_pages,
                 struct radeon_fence **fence);
+void si_dma_vm_set_page(struct radeon_device *rdev,
+                        struct radeon_ib *ib,
+                        uint64_t pe,
+                        uint64_t addr, unsigned count,
+                        uint32_t incr, uint32_t flags);
 void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
 u32 si_get_xclk(struct radeon_device *rdev);
 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev);
@@ -705,6 +706,10 @@ int cik_copy_dma(struct radeon_device *rdev,
                  uint64_t src_offset, uint64_t dst_offset,
                  unsigned num_gpu_pages,
                  struct radeon_fence **fence);
+int cik_copy_cpdma(struct radeon_device *rdev,
+                   uint64_t src_offset, uint64_t dst_offset,
+                   unsigned num_gpu_pages,
+                   struct radeon_fence **fence);
 int cik_sdma_ring_test(struct radeon_device *rdev, struct radeon_ring *ring);
 int cik_sdma_ib_test(struct radeon_device *rdev, struct radeon_ring *ring);
 bool cik_sdma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring);
@@ -731,11 +736,11 @@ int cik_irq_process(struct radeon_device *rdev);
 int cik_vm_init(struct radeon_device *rdev);
 void cik_vm_fini(struct radeon_device *rdev);
 void cik_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
-void cik_vm_set_page(struct radeon_device *rdev,
-                     struct radeon_ib *ib,
-                     uint64_t pe,
-                     uint64_t addr, unsigned count,
-                     uint32_t incr, uint32_t flags);
+void cik_sdma_vm_set_page(struct radeon_device *rdev,
+                          struct radeon_ib *ib,
+                          uint64_t pe,
+                          uint64_t addr, unsigned count,
+                          uint32_t incr, uint32_t flags);
 void cik_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm);
 int cik_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib);
 u32 cik_compute_ring_get_rptr(struct radeon_device *rdev,
diff --git a/drivers/gpu/drm/radeon/radeon_atpx_handler.c b/drivers/gpu/drm/radeon/radeon_atpx_handler.c
index d96070bf8388..6153ec18943a 100644
--- a/drivers/gpu/drm/radeon/radeon_atpx_handler.c
+++ b/drivers/gpu/drm/radeon/radeon_atpx_handler.c
@@ -59,6 +59,10 @@ struct atpx_mux {
         u16 mux;
 } __packed;
 
+bool radeon_is_px(void) {
+        return radeon_atpx_priv.atpx_detected;
+}
+
 /**
  * radeon_atpx_call - call an ATPX method
  *
diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
index 64565732cb98..20a768ac89a8 100644
--- a/drivers/gpu/drm/radeon/radeon_connectors.c
+++ b/drivers/gpu/drm/radeon/radeon_connectors.c
@@ -31,6 +31,8 @@
 #include "radeon.h"
 #include "atom.h"
 
+#include <linux/pm_runtime.h>
+
 extern void
 radeon_combios_connected_scratch_regs(struct drm_connector *connector,
                                       struct drm_encoder *encoder,
@@ -411,6 +413,21 @@ static int radeon_connector_set_property(struct drm_connector *connector, struct
                 }
         }
 
+        if (property == rdev->mode_info.dither_property) {
+                struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+                /* need to find digital encoder on connector */
+                encoder = radeon_find_encoder(connector, DRM_MODE_ENCODER_TMDS);
+                if (!encoder)
+                        return 0;
+
+                radeon_encoder = to_radeon_encoder(encoder);
+
+                if (radeon_connector->dither != val) {
+                        radeon_connector->dither = val;
+                        radeon_property_change_mode(&radeon_encoder->base);
+                }
+        }
+
         if (property == rdev->mode_info.underscan_property) {
                 /* need to find digital encoder on connector */
                 encoder = radeon_find_encoder(connector, DRM_MODE_ENCODER_TMDS);
@@ -626,6 +643,11 @@ radeon_lvds_detect(struct drm_connector *connector, bool force)
         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
         struct drm_encoder *encoder = radeon_best_single_encoder(connector);
         enum drm_connector_status ret = connector_status_disconnected;
+        int r;
+
+        r = pm_runtime_get_sync(connector->dev->dev);
+        if (r < 0)
+                return connector_status_disconnected;
 
         if (encoder) {
                 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
@@ -651,6 +673,8 @@ radeon_lvds_detect(struct drm_connector *connector, bool force)
         /* check acpi lid status ??? */
 
         radeon_connector_update_scratch_regs(connector, ret);
+        pm_runtime_mark_last_busy(connector->dev->dev);
+        pm_runtime_put_autosuspend(connector->dev->dev);
         return ret;
 }
 
@@ -750,6 +774,11 @@ radeon_vga_detect(struct drm_connector *connector, bool force)
         struct drm_encoder_helper_funcs *encoder_funcs;
         bool dret = false;
         enum drm_connector_status ret = connector_status_disconnected;
+        int r;
+
+        r = pm_runtime_get_sync(connector->dev->dev);
+        if (r < 0)
+                return connector_status_disconnected;
 
         encoder = radeon_best_single_encoder(connector);
         if (!encoder)
@@ -790,9 +819,8 @@ radeon_vga_detect(struct drm_connector *connector, bool force)
                          * detected a monitor via load.
                          */
                         if (radeon_connector->detected_by_load)
-                                return connector->status;
-                        else
-                                return ret;
+                                ret = connector->status;
+                        goto out;
                 }
 
                 if (radeon_connector->dac_load_detect && encoder) {
@@ -817,6 +845,11 @@ radeon_vga_detect(struct drm_connector *connector, bool force)
         }
 
         radeon_connector_update_scratch_regs(connector, ret);
+
+out:
+        pm_runtime_mark_last_busy(connector->dev->dev);
+        pm_runtime_put_autosuspend(connector->dev->dev);
+
         return ret;
 }
 
@@ -873,10 +906,15 @@ radeon_tv_detect(struct drm_connector *connector, bool force)
         struct drm_encoder_helper_funcs *encoder_funcs;
         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
         enum drm_connector_status ret = connector_status_disconnected;
+        int r;
 
         if (!radeon_connector->dac_load_detect)
                 return ret;
 
+        r = pm_runtime_get_sync(connector->dev->dev);
+        if (r < 0)
+                return connector_status_disconnected;
+
         encoder = radeon_best_single_encoder(connector);
         if (!encoder)
                 ret = connector_status_disconnected;
@@ -887,6 +925,8 @@ radeon_tv_detect(struct drm_connector *connector, bool force)
         if (ret == connector_status_connected)
                 ret = radeon_connector_analog_encoder_conflict_solve(connector, encoder, ret, false);
         radeon_connector_update_scratch_regs(connector, ret);
+        pm_runtime_mark_last_busy(connector->dev->dev);
+        pm_runtime_put_autosuspend(connector->dev->dev);
         return ret;
 }
 
@@ -954,12 +994,18 @@ radeon_dvi_detect(struct drm_connector *connector, bool force)
         struct drm_encoder *encoder = NULL;
         struct drm_encoder_helper_funcs *encoder_funcs;
         struct drm_mode_object *obj;
-        int i;
+        int i, r;
         enum drm_connector_status ret = connector_status_disconnected;
         bool dret = false, broken_edid = false;
 
-        if (!force && radeon_check_hpd_status_unchanged(connector))
-                return connector->status;
+        r = pm_runtime_get_sync(connector->dev->dev);
+        if (r < 0)
+                return connector_status_disconnected;
+
+        if (!force && radeon_check_hpd_status_unchanged(connector)) {
+                ret = connector->status;
+                goto exit;
+        }
 
         if (radeon_connector->ddc_bus)
                 dret = radeon_ddc_probe(radeon_connector, false);
@@ -1110,6 +1156,11 @@ out:
 
         /* updated in get modes as well since we need to know if it's analog or digital */
         radeon_connector_update_scratch_regs(connector, ret);
+
+exit:
+        pm_runtime_mark_last_busy(connector->dev->dev);
+        pm_runtime_put_autosuspend(connector->dev->dev);
+
         return ret;
 }
 
@@ -1377,9 +1428,16 @@ radeon_dp_detect(struct drm_connector *connector, bool force)
         enum drm_connector_status ret = connector_status_disconnected;
         struct radeon_connector_atom_dig *radeon_dig_connector = radeon_connector->con_priv;
         struct drm_encoder *encoder = radeon_best_single_encoder(connector);
+        int r;
 
-        if (!force && radeon_check_hpd_status_unchanged(connector))
-                return connector->status;
+        r = pm_runtime_get_sync(connector->dev->dev);
+        if (r < 0)
+                return connector_status_disconnected;
+
+        if (!force && radeon_check_hpd_status_unchanged(connector)) {
+                ret = connector->status;
+                goto out;
+        }
 
         if (radeon_connector->edid) {
                 kfree(radeon_connector->edid);
@@ -1443,6 +1501,10 @@ radeon_dp_detect(struct drm_connector *connector, bool force)
         }
 
         radeon_connector_update_scratch_regs(connector, ret);
+out:
+        pm_runtime_mark_last_busy(connector->dev->dev);
+        pm_runtime_put_autosuspend(connector->dev->dev);
+
         return ret;
 }
 
@@ -1658,12 +1720,16 @@ radeon_add_atom_connector(struct drm_device *dev,
                         drm_object_attach_property(&radeon_connector->base.base,
                                                       rdev->mode_info.underscan_vborder_property,
                                                       0);
+
+                        drm_object_attach_property(&radeon_connector->base.base,
+                                                   rdev->mode_info.dither_property,
+                                                   RADEON_FMT_DITHER_DISABLE);
+
                         if (radeon_audio != 0)
                                 drm_object_attach_property(&radeon_connector->base.base,
                                                            rdev->mode_info.audio_property,
-                                                           (radeon_audio == 1) ?
-                                                           RADEON_AUDIO_AUTO :
-                                                           RADEON_AUDIO_DISABLE);
+                                                           RADEON_AUDIO_AUTO);
+
                         subpixel_order = SubPixelHorizontalRGB;
                         connector->interlace_allowed = true;
                         if (connector_type == DRM_MODE_CONNECTOR_HDMIB)
@@ -1760,9 +1826,12 @@ radeon_add_atom_connector(struct drm_device *dev,
                         if (ASIC_IS_DCE2(rdev) && (radeon_audio != 0)) {
                                 drm_object_attach_property(&radeon_connector->base.base,
                                                            rdev->mode_info.audio_property,
-                                                           (radeon_audio == 1) ?
-                                                           RADEON_AUDIO_AUTO :
-                                                           RADEON_AUDIO_DISABLE);
+                                                           RADEON_AUDIO_AUTO);
+                        }
+                        if (ASIC_IS_AVIVO(rdev)) {
+                                drm_object_attach_property(&radeon_connector->base.base,
+                                                           rdev->mode_info.dither_property,
+                                                           RADEON_FMT_DITHER_DISABLE);
                         }
                         if (connector_type == DRM_MODE_CONNECTOR_DVII) {
                                 radeon_connector->dac_load_detect = true;
@@ -1807,9 +1876,12 @@ radeon_add_atom_connector(struct drm_device *dev,
                         if (ASIC_IS_DCE2(rdev) && (radeon_audio != 0)) {
                                 drm_object_attach_property(&radeon_connector->base.base,
                                                            rdev->mode_info.audio_property,
-                                                           (radeon_audio == 1) ?
-                                                           RADEON_AUDIO_AUTO :
-                                                           RADEON_AUDIO_DISABLE);
+                                                           RADEON_AUDIO_AUTO);
+                        }
+                        if (ASIC_IS_AVIVO(rdev)) {
+                                drm_object_attach_property(&radeon_connector->base.base,
+                                                           rdev->mode_info.dither_property,
+                                                           RADEON_FMT_DITHER_DISABLE);
                         }
                         subpixel_order = SubPixelHorizontalRGB;
                         connector->interlace_allowed = true;
@@ -1853,9 +1925,13 @@ radeon_add_atom_connector(struct drm_device *dev,
                         if (ASIC_IS_DCE2(rdev) && (radeon_audio != 0)) {
                                 drm_object_attach_property(&radeon_connector->base.base,
                                                            rdev->mode_info.audio_property,
-                                                           (radeon_audio == 1) ?
-                                                           RADEON_AUDIO_AUTO :
-                                                           RADEON_AUDIO_DISABLE);
+                                                           RADEON_AUDIO_AUTO);
+                        }
+                        if (ASIC_IS_AVIVO(rdev)) {
+                                drm_object_attach_property(&radeon_connector->base.base,
+                                                           rdev->mode_info.dither_property,
+                                                           RADEON_FMT_DITHER_DISABLE);
+
                         }
                         connector->interlace_allowed = true;
                         /* in theory with a DP to VGA converter... */
diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c
index 80285e35bc65..26ca223d12d6 100644
--- a/drivers/gpu/drm/radeon/radeon_cs.c
+++ b/drivers/gpu/drm/radeon/radeon_cs.c
@@ -212,9 +212,7 @@ int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)
                         return -EFAULT;
                 }
                 p->chunks[i].length_dw = user_chunk.length_dw;
-                p->chunks[i].kdata = NULL;
                 p->chunks[i].chunk_id = user_chunk.chunk_id;
-                p->chunks[i].user_ptr = (void __user *)(unsigned long)user_chunk.chunk_data;
                 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) {
                         p->chunk_relocs_idx = i;
                 }
@@ -237,25 +235,31 @@ int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)
                                 return -EINVAL;
                 }
 
-                cdata = (uint32_t *)(unsigned long)user_chunk.chunk_data;
-                if ((p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) ||
-                    (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS)) {
-                        size = p->chunks[i].length_dw * sizeof(uint32_t);
-                        p->chunks[i].kdata = kmalloc(size, GFP_KERNEL);
-                        if (p->chunks[i].kdata == NULL) {
-                                return -ENOMEM;
-                        }
-                        if (DRM_COPY_FROM_USER(p->chunks[i].kdata,
-                                               p->chunks[i].user_ptr, size)) {
-                                return -EFAULT;
-                        }
-                        if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) {
-                                p->cs_flags = p->chunks[i].kdata[0];
-                                if (p->chunks[i].length_dw > 1)
-                                        ring = p->chunks[i].kdata[1];
-                                if (p->chunks[i].length_dw > 2)
-                                        priority = (s32)p->chunks[i].kdata[2];
-                        }
+                size = p->chunks[i].length_dw;
+                cdata = (void __user *)(unsigned long)user_chunk.chunk_data;
+                p->chunks[i].user_ptr = cdata;
+                if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_CONST_IB)
+                        continue;
+
+                if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) {
+                        if (!p->rdev || !(p->rdev->flags & RADEON_IS_AGP))
+                                continue;
+                }
+
+                p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t));
+                size *= sizeof(uint32_t);
+                if (p->chunks[i].kdata == NULL) {
+                        return -ENOMEM;
+                }
+                if (DRM_COPY_FROM_USER(p->chunks[i].kdata, cdata, size)) {
+                        return -EFAULT;
+                }
+                if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) {
+                        p->cs_flags = p->chunks[i].kdata[0];
+                        if (p->chunks[i].length_dw > 1)
+                                ring = p->chunks[i].kdata[1];
+                        if (p->chunks[i].length_dw > 2)
+                                priority = (s32)p->chunks[i].kdata[2];
                 }
         }
 
@@ -278,34 +282,6 @@ int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)
                 }
         }
 
-        /* deal with non-vm */
-        if ((p->chunk_ib_idx != -1) &&
-            ((p->cs_flags & RADEON_CS_USE_VM) == 0) &&
-            (p->chunks[p->chunk_ib_idx].chunk_id == RADEON_CHUNK_ID_IB)) {
-                if (p->chunks[p->chunk_ib_idx].length_dw > (16 * 1024)) {
-                        DRM_ERROR("cs IB too big: %d\n",
-                                  p->chunks[p->chunk_ib_idx].length_dw);
-                        return -EINVAL;
-                }
-                if (p->rdev && (p->rdev->flags & RADEON_IS_AGP)) {
-                        p->chunks[p->chunk_ib_idx].kpage[0] = kmalloc(PAGE_SIZE, GFP_KERNEL);
-                        p->chunks[p->chunk_ib_idx].kpage[1] = kmalloc(PAGE_SIZE, GFP_KERNEL);
-                        if (p->chunks[p->chunk_ib_idx].kpage[0] == NULL ||
-                            p->chunks[p->chunk_ib_idx].kpage[1] == NULL) {
-                                kfree(p->chunks[p->chunk_ib_idx].kpage[0]);
-                                kfree(p->chunks[p->chunk_ib_idx].kpage[1]);
-                                p->chunks[p->chunk_ib_idx].kpage[0] = NULL;
-                                p->chunks[p->chunk_ib_idx].kpage[1] = NULL;
-                                return -ENOMEM;
-                        }
-                }
-                p->chunks[p->chunk_ib_idx].kpage_idx[0] = -1;
-                p->chunks[p->chunk_ib_idx].kpage_idx[1] = -1;
-                p->chunks[p->chunk_ib_idx].last_copied_page = -1;
-                p->chunks[p->chunk_ib_idx].last_page_index =
-                        ((p->chunks[p->chunk_ib_idx].length_dw * 4) - 1) / PAGE_SIZE;
-        }
-
         return 0;
 }
 
@@ -339,13 +315,8 @@ static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error, bo
         kfree(parser->track);
         kfree(parser->relocs);
         kfree(parser->relocs_ptr);
-        for (i = 0; i < parser->nchunks; i++) {
-                kfree(parser->chunks[i].kdata);
-                if ((parser->rdev->flags & RADEON_IS_AGP)) {
-                        kfree(parser->chunks[i].kpage[0]);
-                        kfree(parser->chunks[i].kpage[1]);
-                }
-        }
+        for (i = 0; i < parser->nchunks; i++)
+                drm_free_large(parser->chunks[i].kdata);
         kfree(parser->chunks);
         kfree(parser->chunks_array);
         radeon_ib_free(parser->rdev, &parser->ib);
@@ -355,7 +326,6 @@ static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error, bo
 static int radeon_cs_ib_chunk(struct radeon_device *rdev,
                               struct radeon_cs_parser *parser)
 {
-        struct radeon_cs_chunk *ib_chunk;
         int r;
 
         if (parser->chunk_ib_idx == -1)
@@ -364,28 +334,11 @@ static int radeon_cs_ib_chunk(struct radeon_device *rdev,
         if (parser->cs_flags & RADEON_CS_USE_VM)
                 return 0;
 
-        ib_chunk = &parser->chunks[parser->chunk_ib_idx];
-        /* Copy the packet into the IB, the parser will read from the
-         * input memory (cached) and write to the IB (which can be
-         * uncached).
-         */
-        r =  radeon_ib_get(rdev, parser->ring, &parser->ib,
-                           NULL, ib_chunk->length_dw * 4);
-        if (r) {
-                DRM_ERROR("Failed to get ib !\n");
-                return r;
-        }
-        parser->ib.length_dw = ib_chunk->length_dw;
         r = radeon_cs_parse(rdev, parser->ring, parser);
         if (r || parser->parser_error) {
                 DRM_ERROR("Invalid command stream !\n");
                 return r;
         }
-        r = radeon_cs_finish_pages(parser);
-        if (r) {
-                DRM_ERROR("Invalid command stream !\n");
-                return r;
-        }
 
         if (parser->ring == R600_RING_TYPE_UVD_INDEX)
                 radeon_uvd_note_usage(rdev);
@@ -423,7 +376,6 @@ static int radeon_bo_vm_update_pte(struct radeon_cs_parser *parser,
 static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
                                  struct radeon_cs_parser *parser)
 {
-        struct radeon_cs_chunk *ib_chunk;
         struct radeon_fpriv *fpriv = parser->filp->driver_priv;
         struct radeon_vm *vm = &fpriv->vm;
         int r;
@@ -433,49 +385,13 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
         if ((parser->cs_flags & RADEON_CS_USE_VM) == 0)
                 return 0;
 
-        if ((rdev->family >= CHIP_TAHITI) &&
-            (parser->chunk_const_ib_idx != -1)) {
-                ib_chunk = &parser->chunks[parser->chunk_const_ib_idx];
-                if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
-                        DRM_ERROR("cs IB CONST too big: %d\n", ib_chunk->length_dw);
-                        return -EINVAL;
-                }
-                r =  radeon_ib_get(rdev, parser->ring, &parser->const_ib,
-                                   vm, ib_chunk->length_dw * 4);
-                if (r) {
-                        DRM_ERROR("Failed to get const ib !\n");
-                        return r;
-                }
-                parser->const_ib.is_const_ib = true;
-                parser->const_ib.length_dw = ib_chunk->length_dw;
-                /* Copy the packet into the IB */
-                if (DRM_COPY_FROM_USER(parser->const_ib.ptr, ib_chunk->user_ptr,
-                                       ib_chunk->length_dw * 4)) {
-                        return -EFAULT;
-                }
+        if (parser->const_ib.length_dw) {
                 r = radeon_ring_ib_parse(rdev, parser->ring, &parser->const_ib);
                 if (r) {
                         return r;
                 }
         }
 
-        ib_chunk = &parser->chunks[parser->chunk_ib_idx];
-        if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
-                DRM_ERROR("cs IB too big: %d\n", ib_chunk->length_dw);
-                return -EINVAL;
-        }
-        r =  radeon_ib_get(rdev, parser->ring, &parser->ib,
-                           vm, ib_chunk->length_dw * 4);
-        if (r) {
-                DRM_ERROR("Failed to get ib !\n");
-                return r;
-        }
-        parser->ib.length_dw = ib_chunk->length_dw;
-        /* Copy the packet into the IB */
-        if (DRM_COPY_FROM_USER(parser->ib.ptr, ib_chunk->user_ptr,
-                               ib_chunk->length_dw * 4)) {
-                return -EFAULT;
-        }
         r = radeon_ring_ib_parse(rdev, parser->ring, &parser->ib);
         if (r) {
                 return r;
@@ -527,6 +443,62 @@ static int radeon_cs_handle_lockup(struct radeon_device *rdev, int r)
         return r;
 }
 
+static int radeon_cs_ib_fill(struct radeon_device *rdev, struct radeon_cs_parser *parser)
+{
+        struct radeon_cs_chunk *ib_chunk;
+        struct radeon_vm *vm = NULL;
+        int r;
+
+        if (parser->chunk_ib_idx == -1)
+                return 0;
+
+        if (parser->cs_flags & RADEON_CS_USE_VM) {
+                struct radeon_fpriv *fpriv = parser->filp->driver_priv;
+                vm = &fpriv->vm;
+
+                if ((rdev->family >= CHIP_TAHITI) &&
+                    (parser->chunk_const_ib_idx != -1)) {
+                        ib_chunk = &parser->chunks[parser->chunk_const_ib_idx];
+                        if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
+                                DRM_ERROR("cs IB CONST too big: %d\n", ib_chunk->length_dw);
+                                return -EINVAL;
+                        }
+                        r =  radeon_ib_get(rdev, parser->ring, &parser->const_ib,
+                                           vm, ib_chunk->length_dw * 4);
+                        if (r) {
+                                DRM_ERROR("Failed to get const ib !\n");
+                                return r;
+                        }
+                        parser->const_ib.is_const_ib = true;
+                        parser->const_ib.length_dw = ib_chunk->length_dw;
+                        if (DRM_COPY_FROM_USER(parser->const_ib.ptr,
+                                               ib_chunk->user_ptr,
+                                               ib_chunk->length_dw * 4))
+                                return -EFAULT;
+                }
+
+                ib_chunk = &parser->chunks[parser->chunk_ib_idx];
+                if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
+                        DRM_ERROR("cs IB too big: %d\n", ib_chunk->length_dw);
+                        return -EINVAL;
+                }
+        }
+        ib_chunk = &parser->chunks[parser->chunk_ib_idx];
+
+        r =  radeon_ib_get(rdev, parser->ring, &parser->ib,
+                           vm, ib_chunk->length_dw * 4);
+        if (r) {
+                DRM_ERROR("Failed to get ib !\n");
+                return r;
+        }
+        parser->ib.length_dw = ib_chunk->length_dw;
+        if (ib_chunk->kdata)
+                memcpy(parser->ib.ptr, ib_chunk->kdata, ib_chunk->length_dw * 4);
+        else if (DRM_COPY_FROM_USER(parser->ib.ptr, ib_chunk->user_ptr, ib_chunk->length_dw * 4))
+                return -EFAULT;
+        return 0;
+}
+
 int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 {
         struct radeon_device *rdev = dev->dev_private;
@@ -552,10 +524,15 @@ int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
                 r = radeon_cs_handle_lockup(rdev, r);
                 return r;
         }
-        r = radeon_cs_parser_relocs(&parser);
-        if (r) {
-                if (r != -ERESTARTSYS)
+
+        r = radeon_cs_ib_fill(rdev, &parser);
+        if (!r) {
+                r = radeon_cs_parser_relocs(&parser);
+                if (r && r != -ERESTARTSYS)
                         DRM_ERROR("Failed to parse relocation %d!\n", r);
+        }
+
+        if (r) {
                 radeon_cs_parser_fini(&parser, r, false);
                 up_read(&rdev->exclusive_lock);
                 r = radeon_cs_handle_lockup(rdev, r);
@@ -579,97 +556,6 @@ out:
         return r;
 }
 
-int radeon_cs_finish_pages(struct radeon_cs_parser *p)
-{
-        struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
-        int i;
-        int size = PAGE_SIZE;
-
-        for (i = ibc->last_copied_page + 1; i <= ibc->last_page_index; i++) {
-                if (i == ibc->last_page_index) {
-                        size = (ibc->length_dw * 4) % PAGE_SIZE;
-                        if (size == 0)
-                                size = PAGE_SIZE;
-                }
-                
-                if (DRM_COPY_FROM_USER(p->ib.ptr + (i * (PAGE_SIZE/4)),
-                                       ibc->user_ptr + (i * PAGE_SIZE),
-                                       size))
-                        return -EFAULT;
-        }
-        return 0;
-}
-
-static int radeon_cs_update_pages(struct radeon_cs_parser *p, int pg_idx)
-{
-        int new_page;
-        struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
-        int i;
-        int size = PAGE_SIZE;
-        bool copy1 = (p->rdev && (p->rdev->flags & RADEON_IS_AGP)) ?
-                false : true;
-
-        for (i = ibc->last_copied_page + 1; i < pg_idx; i++) {
-                if (DRM_COPY_FROM_USER(p->ib.ptr + (i * (PAGE_SIZE/4)),
-                                       ibc->user_ptr + (i * PAGE_SIZE),
-                                       PAGE_SIZE)) {
-                        p->parser_error = -EFAULT;
-                        return 0;
-                }
-        }
-
-        if (pg_idx == ibc->last_page_index) {
-                size = (ibc->length_dw * 4) % PAGE_SIZE;
-                if (size == 0)
-                        size = PAGE_SIZE;
-        }
-
-        new_page = ibc->kpage_idx[0] < ibc->kpage_idx[1] ? 0 : 1;
-        if (copy1)
-                ibc->kpage[new_page] = p->ib.ptr + (pg_idx * (PAGE_SIZE / 4));
-
-        if (DRM_COPY_FROM_USER(ibc->kpage[new_page],
-                               ibc->user_ptr + (pg_idx * PAGE_SIZE),
-                               size)) {
-                p->parser_error = -EFAULT;
-                return 0;
-        }
-
-        /* copy to IB for non single case */
-        if (!copy1)
-                memcpy((void *)(p->ib.ptr+(pg_idx*(PAGE_SIZE/4))), ibc->kpage[new_page], size);
-
-        ibc->last_copied_page = pg_idx;
-        ibc->kpage_idx[new_page] = pg_idx;
-
-        return new_page;
-}
-
-u32 radeon_get_ib_value(struct radeon_cs_parser *p, int idx)
-{
-        struct radeon_cs_chunk *ibc = &p->chunks[p->chunk_ib_idx];
-        u32 pg_idx, pg_offset;
-        u32 idx_value = 0;
-        int new_page;
-
-        pg_idx = (idx * 4) / PAGE_SIZE;
-        pg_offset = (idx * 4) % PAGE_SIZE;
-
-        if (ibc->kpage_idx[0] == pg_idx)
-                return ibc->kpage[0][pg_offset/4];
-        if (ibc->kpage_idx[1] == pg_idx)
-                return ibc->kpage[1][pg_offset/4];
-
-        new_page = radeon_cs_update_pages(p, pg_idx);
-        if (new_page < 0) {
-                p->parser_error = new_page;
-                return 0;
-        }
-
-        idx_value = ibc->kpage[new_page][pg_offset/4];
-        return idx_value;
-}
-
 /**
  * radeon_cs_packet_parse() - parse cp packet and point ib index to next packet
  * @parser:     parser structure holding parsing context.
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 841d0e09be3e..b9234c43f43d 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -98,9 +98,16 @@ static const char radeon_family_name[][16] = {
         "BONAIRE",
         "KAVERI",
         "KABINI",
+        "HAWAII",
         "LAST",
 };
 
+#if defined(CONFIG_VGA_SWITCHEROO)
+bool radeon_is_px(void);
+#else
+static inline bool radeon_is_px(void) { return false; }
+#endif
+
 /**
  * radeon_program_register_sequence - program an array of registers.
  *
@@ -1076,7 +1083,10 @@ static bool radeon_switcheroo_quirk_long_wakeup(struct pci_dev *pdev)
 static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
 {
         struct drm_device *dev = pci_get_drvdata(pdev);
-        pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
+
+        if (radeon_is_px() && state == VGA_SWITCHEROO_OFF)
+                return;
+
         if (state == VGA_SWITCHEROO_ON) {
                 unsigned d3_delay = dev->pdev->d3_delay;
 
@@ -1087,7 +1097,7 @@ static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switchero
                 if (d3_delay < 20 && radeon_switcheroo_quirk_long_wakeup(pdev))
                         dev->pdev->d3_delay = 20;
 
-                radeon_resume_kms(dev);
+                radeon_resume_kms(dev, true, true);
 
                 dev->pdev->d3_delay = d3_delay;
 
@@ -1097,7 +1107,7 @@ static void radeon_switcheroo_set_state(struct pci_dev *pdev, enum vga_switchero
                 printk(KERN_INFO "radeon: switched off\n");
                 drm_kms_helper_poll_disable(dev);
                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
-                radeon_suspend_kms(dev, pmm);
+                radeon_suspend_kms(dev, true, true);
                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
         }
 }
@@ -1147,6 +1157,7 @@ int radeon_device_init(struct radeon_device *rdev,
 {
         int r, i;
         int dma_bits;
+        bool runtime = false;
 
         rdev->shutdown = false;
         rdev->dev = &pdev->dev;
@@ -1293,7 +1304,14 @@ int radeon_device_init(struct radeon_device *rdev,
         /* this will fail for cards that aren't VGA class devices, just
          * ignore it */
         vga_client_register(rdev->pdev, rdev, NULL, radeon_vga_set_decode);
-        vga_switcheroo_register_client(rdev->pdev, &radeon_switcheroo_ops, false);
+
+        if (radeon_runtime_pm == 1)
+                runtime = true;
+        if ((radeon_runtime_pm == -1) && radeon_is_px())
+                runtime = true;
+        vga_switcheroo_register_client(rdev->pdev, &radeon_switcheroo_ops, runtime);
+        if (runtime)
+                vga_switcheroo_init_domain_pm_ops(rdev->dev, &rdev->vga_pm_domain);
 
         r = radeon_init(rdev);
         if (r)
@@ -1383,7 +1401,7 @@ void radeon_device_fini(struct radeon_device *rdev)
  * Returns 0 for success or an error on failure.
  * Called at driver suspend.
  */
-int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)
+int radeon_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon)
 {
         struct radeon_device *rdev;
         struct drm_crtc *crtc;
@@ -1394,9 +1412,7 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)
         if (dev == NULL || dev->dev_private == NULL) {
                 return -ENODEV;
         }
-        if (state.event == PM_EVENT_PRETHAW) {
-                return 0;
-        }
+
         rdev = dev->dev_private;
 
         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
@@ -1455,14 +1471,17 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)
         radeon_agp_suspend(rdev);
 
         pci_save_state(dev->pdev);
-        if (state.event == PM_EVENT_SUSPEND) {
+        if (suspend) {
                 /* Shut down the device */
                 pci_disable_device(dev->pdev);
                 pci_set_power_state(dev->pdev, PCI_D3hot);
         }
-        console_lock();
-        radeon_fbdev_set_suspend(rdev, 1);
-        console_unlock();
+
+        if (fbcon) {
+                console_lock();
+                radeon_fbdev_set_suspend(rdev, 1);
+                console_unlock();
+        }
         return 0;
 }
 
@@ -1475,7 +1494,7 @@ int radeon_suspend_kms(struct drm_device *dev, pm_message_t state)
  * Returns 0 for success or an error on failure.
  * Called at driver resume.
  */
-int radeon_resume_kms(struct drm_device *dev)
+int radeon_resume_kms(struct drm_device *dev, bool resume, bool fbcon)
 {
         struct drm_connector *connector;
         struct radeon_device *rdev = dev->dev_private;
@@ -1484,12 +1503,17 @@ int radeon_resume_kms(struct drm_device *dev)
         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
                 return 0;
 
-        console_lock();
-        pci_set_power_state(dev->pdev, PCI_D0);
-        pci_restore_state(dev->pdev);
-        if (pci_enable_device(dev->pdev)) {
-                console_unlock();
-                return -1;
+        if (fbcon) {
+                console_lock();
+        }
+        if (resume) {
+                pci_set_power_state(dev->pdev, PCI_D0);
+                pci_restore_state(dev->pdev);
+                if (pci_enable_device(dev->pdev)) {
+                        if (fbcon)
+                                console_unlock();
+                        return -1;
+                }
         }
         /* resume AGP if in use */
         radeon_agp_resume(rdev);
@@ -1502,9 +1526,11 @@ int radeon_resume_kms(struct drm_device *dev)
         radeon_pm_resume(rdev);
         radeon_restore_bios_scratch_regs(rdev);
 
-        radeon_fbdev_set_suspend(rdev, 0);
-        console_unlock();
-
+        if (fbcon) {
+                radeon_fbdev_set_suspend(rdev, 0);
+                console_unlock();
+        }
+       
         /* init dig PHYs, disp eng pll */
         if (rdev->is_atom_bios) {
                 radeon_atom_encoder_init(rdev);
@@ -1549,6 +1575,14 @@ int radeon_gpu_reset(struct radeon_device *rdev)
         int resched;
 
         down_write(&rdev->exclusive_lock);
+
+        if (!rdev->needs_reset) {
+                up_write(&rdev->exclusive_lock);
+                return 0;
+        }
+
+        rdev->needs_reset = false;
+
         radeon_save_bios_scratch_regs(rdev);
         /* block TTM */
         resched = ttm_bo_lock_delayed_workqueue(&rdev->mman.bdev);
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index 0d1aa050d41d..7b253815a323 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -30,6 +30,7 @@
 #include "atom.h"
 #include <asm/div64.h>
 
+#include <linux/pm_runtime.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
 
@@ -306,7 +307,7 @@ void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id)
          */
         if (update_pending &&
             (DRM_SCANOUTPOS_VALID & radeon_get_crtc_scanoutpos(rdev->ddev, crtc_id,
-                                                               &vpos, &hpos)) &&
+                                                               &vpos, &hpos, NULL, NULL)) &&
             ((vpos >= (99 * rdev->mode_info.crtcs[crtc_id]->base.hwmode.crtc_vdisplay)/100) ||
              (vpos < 0 && !ASIC_IS_AVIVO(rdev)))) {
                 /* crtc didn't flip in this target vblank interval,
@@ -494,11 +495,55 @@ unlock_free:
         return r;
 }
 
+static int
+radeon_crtc_set_config(struct drm_mode_set *set)
+{
+        struct drm_device *dev;
+        struct radeon_device *rdev;
+        struct drm_crtc *crtc;
+        bool active = false;
+        int ret;
+
+        if (!set || !set->crtc)
+                return -EINVAL;
+
+        dev = set->crtc->dev;
+
+        ret = pm_runtime_get_sync(dev->dev);
+        if (ret < 0)
+                return ret;
+
+        ret = drm_crtc_helper_set_config(set);
+
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
+                if (crtc->enabled)
+                        active = true;
+
+        pm_runtime_mark_last_busy(dev->dev);
+
+        rdev = dev->dev_private;
+        /* if we have active crtcs and we don't have a power ref,
+           take the current one */
+        if (active && !rdev->have_disp_power_ref) {
+                rdev->have_disp_power_ref = true;
+                return ret;
+        }
+        /* if we have no active crtcs, then drop the power ref
+           we got before */
+        if (!active && rdev->have_disp_power_ref) {
+                pm_runtime_put_autosuspend(dev->dev);
+                rdev->have_disp_power_ref = false;
+        }
+
+        /* drop the power reference we got coming in here */
+        pm_runtime_put_autosuspend(dev->dev);
+        return ret;
+}
 static const struct drm_crtc_funcs radeon_crtc_funcs = {
         .cursor_set = radeon_crtc_cursor_set,
         .cursor_move = radeon_crtc_cursor_move,
         .gamma_set = radeon_crtc_gamma_set,
-        .set_config = drm_crtc_helper_set_config,
+        .set_config = radeon_crtc_set_config,
         .destroy = radeon_crtc_destroy,
         .page_flip = radeon_crtc_page_flip,
 };
@@ -1178,6 +1223,12 @@ static struct drm_prop_enum_list radeon_audio_enum_list[] =
         { RADEON_AUDIO_AUTO, "auto" },
 };
 
+/* XXX support different dither options? spatial, temporal, both, etc. */
+static struct drm_prop_enum_list radeon_dither_enum_list[] =
+{       { RADEON_FMT_DITHER_DISABLE, "off" },
+        { RADEON_FMT_DITHER_ENABLE, "on" },
+};
+
 static int radeon_modeset_create_props(struct radeon_device *rdev)
 {
         int sz;
@@ -1234,6 +1285,12 @@ static int radeon_modeset_create_props(struct radeon_device *rdev)
                                          "audio",
                                          radeon_audio_enum_list, sz);
 
+        sz = ARRAY_SIZE(radeon_dither_enum_list);
+        rdev->mode_info.dither_property =
+                drm_property_create_enum(rdev->ddev, 0,
+                                         "dither",
+                                         radeon_dither_enum_list, sz);
+
         return 0;
 }
 
@@ -1539,12 +1596,17 @@ bool radeon_crtc_scaling_mode_fixup(struct drm_crtc *crtc,
 }
 
 /*
- * Retrieve current video scanout position of crtc on a given gpu.
+ * Retrieve current video scanout position of crtc on a given gpu, and
+ * an optional accurate timestamp of when query happened.
  *
  * \param dev Device to query.
  * \param crtc Crtc to query.
  * \param *vpos Location where vertical scanout position should be stored.
  * \param *hpos Location where horizontal scanout position should go.
+ * \param *stime Target location for timestamp taken immediately before
+ *               scanout position query. Can be NULL to skip timestamp.
+ * \param *etime Target location for timestamp taken immediately after
+ *               scanout position query. Can be NULL to skip timestamp.
  *
  * Returns vpos as a positive number while in active scanout area.
  * Returns vpos as a negative number inside vblank, counting the number
@@ -1560,7 +1622,8 @@ bool radeon_crtc_scaling_mode_fixup(struct drm_crtc *crtc,
  * unknown small number of scanlines wrt. real scanout position.
  *
  */
-int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc, int *vpos, int *hpos)
+int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc, int *vpos, int *hpos,
+                               ktime_t *stime, ktime_t *etime)
 {
         u32 stat_crtc = 0, vbl = 0, position = 0;
         int vbl_start, vbl_end, vtotal, ret = 0;
@@ -1568,6 +1631,12 @@ int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc, int *vpos, int
 
         struct radeon_device *rdev = dev->dev_private;
 
+        /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
+
+        /* Get optional system timestamp before query. */
+        if (stime)
+                *stime = ktime_get();
+
         if (ASIC_IS_DCE4(rdev)) {
                 if (crtc == 0) {
                         vbl = RREG32(EVERGREEN_CRTC_V_BLANK_START_END +
@@ -1650,6 +1719,12 @@ int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc, int *vpos, int
                 }
         }
 
+        /* Get optional system timestamp after query. */
+        if (etime)
+                *etime = ktime_get();
+
+        /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
+
         /* Decode into vertical and horizontal scanout position. */
         *vpos = position & 0x1fff;
         *hpos = (position >> 16) & 0x1fff;
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index b01f231c2f19..1aee32213f66 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -36,8 +36,9 @@
 #include <drm/drm_pciids.h>
 #include <linux/console.h>
 #include <linux/module.h>
-
-
+#include <linux/pm_runtime.h>
+#include <linux/vga_switcheroo.h>
+#include "drm_crtc_helper.h"
 /*
  * KMS wrapper.
  * - 2.0.0 - initial interface
@@ -87,8 +88,8 @@ void radeon_driver_postclose_kms(struct drm_device *dev,
                                  struct drm_file *file_priv);
 void radeon_driver_preclose_kms(struct drm_device *dev,
                                 struct drm_file *file_priv);
-int radeon_suspend_kms(struct drm_device *dev, pm_message_t state);
-int radeon_resume_kms(struct drm_device *dev);
+int radeon_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon);
+int radeon_resume_kms(struct drm_device *dev, bool resume, bool fbcon);
 u32 radeon_get_vblank_counter_kms(struct drm_device *dev, int crtc);
 int radeon_enable_vblank_kms(struct drm_device *dev, int crtc);
 void radeon_disable_vblank_kms(struct drm_device *dev, int crtc);
@@ -106,7 +107,8 @@ int radeon_gem_object_open(struct drm_gem_object *obj,
 void radeon_gem_object_close(struct drm_gem_object *obj,
                                 struct drm_file *file_priv);
 extern int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc,
-                                      int *vpos, int *hpos);
+                                      int *vpos, int *hpos, ktime_t *stime,
+                                      ktime_t *etime);
 extern const struct drm_ioctl_desc radeon_ioctls_kms[];
 extern int radeon_max_kms_ioctl;
 int radeon_mmap(struct file *filp, struct vm_area_struct *vma);
@@ -136,9 +138,11 @@ void radeon_debugfs_cleanup(struct drm_minor *minor);
 #if defined(CONFIG_VGA_SWITCHEROO)
 void radeon_register_atpx_handler(void);
 void radeon_unregister_atpx_handler(void);
+bool radeon_is_px(void);
 #else
 static inline void radeon_register_atpx_handler(void) {}
 static inline void radeon_unregister_atpx_handler(void) {}
+static inline bool radeon_is_px(void) { return false; }
 #endif
 
 int radeon_no_wb;
@@ -161,6 +165,7 @@ int radeon_lockup_timeout = 10000;
 int radeon_fastfb = 0;
 int radeon_dpm = -1;
 int radeon_aspm = -1;
+int radeon_runtime_pm = -1;
 
 MODULE_PARM_DESC(no_wb, "Disable AGP writeback for scratch registers");
 module_param_named(no_wb, radeon_no_wb, int, 0444);
@@ -222,6 +227,9 @@ module_param_named(dpm, radeon_dpm, int, 0444);
 MODULE_PARM_DESC(aspm, "ASPM support (1 = enable, 0 = disable, -1 = auto)");
 module_param_named(aspm, radeon_aspm, int, 0444);
 
+MODULE_PARM_DESC(runpm, "PX runtime pm (1 = force enable, 0 = disable, -1 = PX only default)");
+module_param_named(runpm, radeon_runtime_pm, int, 0444);
+
 static struct pci_device_id pciidlist[] = {
         radeon_PCI_IDS
 };
@@ -258,6 +266,7 @@ static int radeon_resume(struct drm_device *dev)
         return 0;
 }
 
+
 static const struct file_operations radeon_driver_old_fops = {
         .owner = THIS_MODULE,
         .open = drm_open,
@@ -352,25 +361,144 @@ radeon_pci_remove(struct pci_dev *pdev)
         drm_put_dev(dev);
 }
 
-static int
-radeon_pci_suspend(struct pci_dev *pdev, pm_message_t state)
+static int radeon_pmops_suspend(struct device *dev)
 {
-        struct drm_device *dev = pci_get_drvdata(pdev);
-        return radeon_suspend_kms(dev, state);
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+        return radeon_suspend_kms(drm_dev, true, true);
 }
 
-static int
-radeon_pci_resume(struct pci_dev *pdev)
+static int radeon_pmops_resume(struct device *dev)
 {
-        struct drm_device *dev = pci_get_drvdata(pdev);
-        return radeon_resume_kms(dev);
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+        return radeon_resume_kms(drm_dev, true, true);
+}
+
+static int radeon_pmops_freeze(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+        return radeon_suspend_kms(drm_dev, false, true);
+}
+
+static int radeon_pmops_thaw(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+        return radeon_resume_kms(drm_dev, false, true);
+}
+
+static int radeon_pmops_runtime_suspend(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+        int ret;
+
+        if (radeon_runtime_pm == 0)
+                return -EINVAL;
+
+        drm_dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
+        drm_kms_helper_poll_disable(drm_dev);
+        vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_OFF);
+
+        ret = radeon_suspend_kms(drm_dev, false, false);
+        pci_save_state(pdev);
+        pci_disable_device(pdev);
+        pci_set_power_state(pdev, PCI_D3cold);
+        drm_dev->switch_power_state = DRM_SWITCH_POWER_DYNAMIC_OFF;
+
+        return 0;
+}
+
+static int radeon_pmops_runtime_resume(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+        int ret;
+
+        if (radeon_runtime_pm == 0)
+                return -EINVAL;
+
+        drm_dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
+
+        pci_set_power_state(pdev, PCI_D0);
+        pci_restore_state(pdev);
+        ret = pci_enable_device(pdev);
+        if (ret)
+                return ret;
+        pci_set_master(pdev);
+
+        ret = radeon_resume_kms(drm_dev, false, false);
+        drm_kms_helper_poll_enable(drm_dev);
+        vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_ON);
+        drm_dev->switch_power_state = DRM_SWITCH_POWER_ON;
+        return 0;
 }
 
+static int radeon_pmops_runtime_idle(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+        struct drm_crtc *crtc;
+
+        if (radeon_runtime_pm == 0)
+                return -EBUSY;
+
+        /* are we PX enabled? */
+        if (radeon_runtime_pm == -1 && !radeon_is_px()) {
+                DRM_DEBUG_DRIVER("failing to power off - not px\n");
+                return -EBUSY;
+        }
+
+        list_for_each_entry(crtc, &drm_dev->mode_config.crtc_list, head) {
+                if (crtc->enabled) {
+                        DRM_DEBUG_DRIVER("failing to power off - crtc active\n");
+                        return -EBUSY;
+                }
+        }
+
+        pm_runtime_mark_last_busy(dev);
+        pm_runtime_autosuspend(dev);
+        /* we don't want the main rpm_idle to call suspend - we want to autosuspend */
+        return 1;
+}
+
+long radeon_drm_ioctl(struct file *filp,
+                      unsigned int cmd, unsigned long arg)
+{
+        struct drm_file *file_priv = filp->private_data;
+        struct drm_device *dev;
+        long ret;
+        dev = file_priv->minor->dev;
+        ret = pm_runtime_get_sync(dev->dev);
+        if (ret < 0)
+                return ret;
+
+        ret = drm_ioctl(filp, cmd, arg);
+        
+        pm_runtime_mark_last_busy(dev->dev);
+        pm_runtime_put_autosuspend(dev->dev);
+        return ret;
+}
+
+static const struct dev_pm_ops radeon_pm_ops = {
+        .suspend = radeon_pmops_suspend,
+        .resume = radeon_pmops_resume,
+        .freeze = radeon_pmops_freeze,
+        .thaw = radeon_pmops_thaw,
+        .poweroff = radeon_pmops_freeze,
+        .restore = radeon_pmops_resume,
+        .runtime_suspend = radeon_pmops_runtime_suspend,
+        .runtime_resume = radeon_pmops_runtime_resume,
+        .runtime_idle = radeon_pmops_runtime_idle,
+};
+
 static const struct file_operations radeon_driver_kms_fops = {
         .owner = THIS_MODULE,
         .open = drm_open,
         .release = drm_release,
-        .unlocked_ioctl = drm_ioctl,
+        .unlocked_ioctl = radeon_drm_ioctl,
         .mmap = radeon_mmap,
         .poll = drm_poll,
         .read = drm_read,
@@ -379,6 +507,15 @@ static const struct file_operations radeon_driver_kms_fops = {
 #endif
 };
 
+
+static void
+radeon_pci_shutdown(struct pci_dev *pdev)
+{
+        struct drm_device *dev = pci_get_drvdata(pdev);
+
+        radeon_driver_unload_kms(dev);
+}
+
 static struct drm_driver kms_driver = {
         .driver_features =
             DRIVER_USE_AGP |
@@ -391,8 +528,6 @@ static struct drm_driver kms_driver = {
         .postclose = radeon_driver_postclose_kms,
         .lastclose = radeon_driver_lastclose_kms,
         .unload = radeon_driver_unload_kms,
-        .suspend = radeon_suspend_kms,
-        .resume = radeon_resume_kms,
         .get_vblank_counter = radeon_get_vblank_counter_kms,
         .enable_vblank = radeon_enable_vblank_kms,
         .disable_vblank = radeon_disable_vblank_kms,
@@ -449,8 +584,8 @@ static struct pci_driver radeon_kms_pci_driver = {
         .id_table = pciidlist,
         .probe = radeon_pci_probe,
         .remove = radeon_pci_remove,
-        .suspend = radeon_pci_suspend,
-        .resume = radeon_pci_resume,
+        .driver.pm = &radeon_pm_ops,
+        .shutdown = radeon_pci_shutdown,
 };
 
 static int __init radeon_init(void)
diff --git a/drivers/gpu/drm/radeon/radeon_drv.h b/drivers/gpu/drm/radeon/radeon_drv.h
index b369d42f7de5..543dcfae7e6f 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.h
+++ b/drivers/gpu/drm/radeon/radeon_drv.h
@@ -113,6 +113,9 @@
 #define DRIVER_MINOR            33
 #define DRIVER_PATCHLEVEL       0
 
+long radeon_drm_ioctl(struct file *filp,
+                      unsigned int cmd, unsigned long arg);
+
 /* The rest of the file is DEPRECATED! */
 #ifdef CONFIG_DRM_RADEON_UMS
 
diff --git a/drivers/gpu/drm/radeon/radeon_family.h b/drivers/gpu/drm/radeon/radeon_family.h
index 3c8289083f9d..614ad549297f 100644
--- a/drivers/gpu/drm/radeon/radeon_family.h
+++ b/drivers/gpu/drm/radeon/radeon_family.h
@@ -96,6 +96,7 @@ enum radeon_family {
         CHIP_BONAIRE,
         CHIP_KAVERI,
         CHIP_KABINI,
+        CHIP_HAWAII,
         CHIP_LAST,
 };
 
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c
index ddb8f8e04eb5..281d14c22a47 100644
--- a/drivers/gpu/drm/radeon/radeon_fence.c
+++ b/drivers/gpu/drm/radeon/radeon_fence.c
@@ -190,10 +190,8 @@ void radeon_fence_process(struct radeon_device *rdev, int ring)
                 }
         } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq);
 
-        if (wake) {
-                rdev->fence_drv[ring].last_activity = jiffies;
+        if (wake)
                 wake_up_all(&rdev->fence_queue);
-        }
 }
 
 /**
@@ -212,13 +210,13 @@ static void radeon_fence_destroy(struct kref *kref)
 }
 
 /**
- * radeon_fence_seq_signaled - check if a fence sequeuce number has signaled
+ * radeon_fence_seq_signaled - check if a fence sequence number has signaled
  *
  * @rdev: radeon device pointer
  * @seq: sequence number
  * @ring: ring index the fence is associated with
  *
- * Check if the last singled fence sequnce number is >= the requested
+ * Check if the last signaled fence sequnce number is >= the requested
  * sequence number (all asics).
  * Returns true if the fence has signaled (current fence value
  * is >= requested value) or false if it has not (current fence
@@ -263,113 +261,131 @@ bool radeon_fence_signaled(struct radeon_fence *fence)
 }
 
 /**
- * radeon_fence_wait_seq - wait for a specific sequence number
+ * radeon_fence_any_seq_signaled - check if any sequence number is signaled
  *
  * @rdev: radeon device pointer
- * @target_seq: sequence number we want to wait for
- * @ring: ring index the fence is associated with
+ * @seq: sequence numbers
+ *
+ * Check if the last signaled fence sequnce number is >= the requested
+ * sequence number (all asics).
+ * Returns true if any has signaled (current value is >= requested value)
+ * or false if it has not. Helper function for radeon_fence_wait_seq.
+ */
+static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq)
+{
+        unsigned i;
+
+        for (i = 0; i < RADEON_NUM_RINGS; ++i) {
+                if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i))
+                        return true;
+        }
+        return false;
+}
+
+/**
+ * radeon_fence_wait_seq - wait for a specific sequence numbers
+ *
+ * @rdev: radeon device pointer
+ * @target_seq: sequence number(s) we want to wait for
  * @intr: use interruptable sleep
  * @lock_ring: whether the ring should be locked or not
  *
- * Wait for the requested sequence number to be written (all asics).
+ * Wait for the requested sequence number(s) to be written by any ring
+ * (all asics).  Sequnce number array is indexed by ring id.
  * @intr selects whether to use interruptable (true) or non-interruptable
  * (false) sleep when waiting for the sequence number.  Helper function
- * for radeon_fence_wait(), et al.
+ * for radeon_fence_wait_*().
  * Returns 0 if the sequence number has passed, error for all other cases.
- * -EDEADLK is returned when a GPU lockup has been detected and the ring is
- * marked as not ready so no further jobs get scheduled until a successful
- * reset.
+ * -EDEADLK is returned when a GPU lockup has been detected.
  */
-static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 target_seq,
-                                 unsigned ring, bool intr, bool lock_ring)
+static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 *target_seq,
+                                 bool intr, bool lock_ring)
 {
-        unsigned long timeout, last_activity;
-        uint64_t seq;
-        unsigned i;
+        uint64_t last_seq[RADEON_NUM_RINGS];
         bool signaled;
-        int r;
+        int i, r;
+
+        while (!radeon_fence_any_seq_signaled(rdev, target_seq)) {
+
+                /* Save current sequence values, used to check for GPU lockups */
+                for (i = 0; i < RADEON_NUM_RINGS; ++i) {
+                        if (!target_seq[i])
+                                continue;
 
-        while (target_seq > atomic64_read(&rdev->fence_drv[ring].last_seq)) {
-                if (!rdev->ring[ring].ready) {
-                        return -EBUSY;
+                        last_seq[i] = atomic64_read(&rdev->fence_drv[i].last_seq);
+                        trace_radeon_fence_wait_begin(rdev->ddev, target_seq[i]);
+                        radeon_irq_kms_sw_irq_get(rdev, i);
                 }
 
-                timeout = jiffies - RADEON_FENCE_JIFFIES_TIMEOUT;
-                if (time_after(rdev->fence_drv[ring].last_activity, timeout)) {
-                        /* the normal case, timeout is somewhere before last_activity */
-                        timeout = rdev->fence_drv[ring].last_activity - timeout;
+                if (intr) {
+                        r = wait_event_interruptible_timeout(rdev->fence_queue, (
+                                (signaled = radeon_fence_any_seq_signaled(rdev, target_seq))
+                                 || rdev->needs_reset), RADEON_FENCE_JIFFIES_TIMEOUT);
                 } else {
-                        /* either jiffies wrapped around, or no fence was signaled in the last 500ms
-                         * anyway we will just wait for the minimum amount and then check for a lockup
-                         */
-                        timeout = 1;
+                        r = wait_event_timeout(rdev->fence_queue, (
+                                (signaled = radeon_fence_any_seq_signaled(rdev, target_seq))
+                                 || rdev->needs_reset), RADEON_FENCE_JIFFIES_TIMEOUT);
                 }
-                seq = atomic64_read(&rdev->fence_drv[ring].last_seq);
-                /* Save current last activity valuee, used to check for GPU lockups */
-                last_activity = rdev->fence_drv[ring].last_activity;
 
-                trace_radeon_fence_wait_begin(rdev->ddev, seq);
-                radeon_irq_kms_sw_irq_get(rdev, ring);
-                if (intr) {
-                        r = wait_event_interruptible_timeout(rdev->fence_queue,
-                                (signaled = radeon_fence_seq_signaled(rdev, target_seq, ring)),
-                                timeout);
-                } else {
-                        r = wait_event_timeout(rdev->fence_queue,
-                                (signaled = radeon_fence_seq_signaled(rdev, target_seq, ring)),
-                                timeout);
+                for (i = 0; i < RADEON_NUM_RINGS; ++i) {
+                        if (!target_seq[i])
+                                continue;
+
+                        radeon_irq_kms_sw_irq_put(rdev, i);
+                        trace_radeon_fence_wait_end(rdev->ddev, target_seq[i]);
                 }
-                radeon_irq_kms_sw_irq_put(rdev, ring);
-                if (unlikely(r < 0)) {
+
+                if (unlikely(r < 0))
                         return r;
-                }
-                trace_radeon_fence_wait_end(rdev->ddev, seq);
 
                 if (unlikely(!signaled)) {
+                        if (rdev->needs_reset)
+                                return -EDEADLK;
+
                         /* we were interrupted for some reason and fence
                          * isn't signaled yet, resume waiting */
-                        if (r) {
+                        if (r)
                                 continue;
+
+                        for (i = 0; i < RADEON_NUM_RINGS; ++i) {
+                                if (!target_seq[i])
+                                        continue;
+
+                                if (last_seq[i] != atomic64_read(&rdev->fence_drv[i].last_seq))
+                                        break;
                         }
 
-                        /* check if sequence value has changed since last_activity */
-                        if (seq != atomic64_read(&rdev->fence_drv[ring].last_seq)) {
+                        if (i != RADEON_NUM_RINGS)
                                 continue;
-                        }
 
-                        if (lock_ring) {
+                        if (lock_ring)
                                 mutex_lock(&rdev->ring_lock);
-                        }
 
-                        /* test if somebody else has already decided that this is a lockup */
-                        if (last_activity != rdev->fence_drv[ring].last_activity) {
-                                if (lock_ring) {
-                                        mutex_unlock(&rdev->ring_lock);
-                                }
-                                continue;
+                        for (i = 0; i < RADEON_NUM_RINGS; ++i) {
+                                if (!target_seq[i])
+                                        continue;
+
+                                if (radeon_ring_is_lockup(rdev, i, &rdev->ring[i]))
+                                        break;
                         }
 
-                        if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) {
+                        if (i < RADEON_NUM_RINGS) {
                                 /* good news we believe it's a lockup */
-                                dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx last fence id 0x%016llx)\n",
-                                         target_seq, seq);
-
-                                /* change last activity so nobody else think there is a lockup */
-                                for (i = 0; i < RADEON_NUM_RINGS; ++i) {
-                                        rdev->fence_drv[i].last_activity = jiffies;
-                                }
-
-                                /* mark the ring as not ready any more */
-                                rdev->ring[ring].ready = false;
-                                if (lock_ring) {
+                                dev_warn(rdev->dev, "GPU lockup (waiting for "
+                                         "0x%016llx last fence id 0x%016llx on"
+                                         " ring %d)\n",
+                                         target_seq[i], last_seq[i], i);
+
+                                /* remember that we need an reset */
+                                rdev->needs_reset = true;
+                                if (lock_ring)
                                         mutex_unlock(&rdev->ring_lock);
-                                }
+                                wake_up_all(&rdev->fence_queue);
                                 return -EDEADLK;
                         }
 
-                        if (lock_ring) {
+                        if (lock_ring)
                                 mutex_unlock(&rdev->ring_lock);
-                        }
                 }
         }
         return 0;
@@ -388,6 +404,7 @@ static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 target_seq,
  */
 int radeon_fence_wait(struct radeon_fence *fence, bool intr)
 {
+        uint64_t seq[RADEON_NUM_RINGS] = {};
         int r;
 
         if (fence == NULL) {
@@ -395,147 +412,15 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr)
                 return -EINVAL;
         }
 
-        r = radeon_fence_wait_seq(fence->rdev, fence->seq,
-                                  fence->ring, intr, true);
-        if (r) {
-                return r;
-        }
-        fence->seq = RADEON_FENCE_SIGNALED_SEQ;
-        return 0;
-}
-
-static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq)
-{
-        unsigned i;
-
-        for (i = 0; i < RADEON_NUM_RINGS; ++i) {
-                if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i)) {
-                        return true;
-                }
-        }
-        return false;
-}
+        seq[fence->ring] = fence->seq;
+        if (seq[fence->ring] == RADEON_FENCE_SIGNALED_SEQ)
+                return 0;
 
-/**
- * radeon_fence_wait_any_seq - wait for a sequence number on any ring
- *
- * @rdev: radeon device pointer
- * @target_seq: sequence number(s) we want to wait for
- * @intr: use interruptable sleep
- *
- * Wait for the requested sequence number(s) to be written by any ring
- * (all asics).  Sequnce number array is indexed by ring id.
- * @intr selects whether to use interruptable (true) or non-interruptable
- * (false) sleep when waiting for the sequence number.  Helper function
- * for radeon_fence_wait_any(), et al.
- * Returns 0 if the sequence number has passed, error for all other cases.
- */
-static int radeon_fence_wait_any_seq(struct radeon_device *rdev,
-                                     u64 *target_seq, bool intr)
-{
-        unsigned long timeout, last_activity, tmp;
-        unsigned i, ring = RADEON_NUM_RINGS;
-        bool signaled;
-        int r;
-
-        for (i = 0, last_activity = 0; i < RADEON_NUM_RINGS; ++i) {
-                if (!target_seq[i]) {
-                        continue;
-                }
-
-                /* use the most recent one as indicator */
-                if (time_after(rdev->fence_drv[i].last_activity, last_activity)) {
-                        last_activity = rdev->fence_drv[i].last_activity;
-                }
-
-                /* For lockup detection just pick the lowest ring we are
-                 * actively waiting for
-                 */
-                if (i < ring) {
-                        ring = i;
-                }
-        }
-
-        /* nothing to wait for ? */
-        if (ring == RADEON_NUM_RINGS) {
-                return -ENOENT;
-        }
-
-        while (!radeon_fence_any_seq_signaled(rdev, target_seq)) {
-                timeout = jiffies - RADEON_FENCE_JIFFIES_TIMEOUT;
-                if (time_after(last_activity, timeout)) {
-                        /* the normal case, timeout is somewhere before last_activity */
-                        timeout = last_activity - timeout;
-                } else {
-                        /* either jiffies wrapped around, or no fence was signaled in the last 500ms
-                         * anyway we will just wait for the minimum amount and then check for a lockup
-                         */
-                        timeout = 1;
-                }
-
-                trace_radeon_fence_wait_begin(rdev->ddev, target_seq[ring]);
-                for (i = 0; i < RADEON_NUM_RINGS; ++i) {
-                        if (target_seq[i]) {
-                                radeon_irq_kms_sw_irq_get(rdev, i);
-                        }
-                }
-                if (intr) {
-                        r = wait_event_interruptible_timeout(rdev->fence_queue,
-                                (signaled = radeon_fence_any_seq_signaled(rdev, target_seq)),
-                                timeout);
-                } else {
-                        r = wait_event_timeout(rdev->fence_queue,
-                                (signaled = radeon_fence_any_seq_signaled(rdev, target_seq)),
-                                timeout);
-                }
-                for (i = 0; i < RADEON_NUM_RINGS; ++i) {
-                        if (target_seq[i]) {
-                                radeon_irq_kms_sw_irq_put(rdev, i);
-                        }
-                }
-                if (unlikely(r < 0)) {
-                        return r;
-                }
-                trace_radeon_fence_wait_end(rdev->ddev, target_seq[ring]);
-
-                if (unlikely(!signaled)) {
-                        /* we were interrupted for some reason and fence
-                         * isn't signaled yet, resume waiting */
-                        if (r) {
-                                continue;
-                        }
-
-                        mutex_lock(&rdev->ring_lock);
-                        for (i = 0, tmp = 0; i < RADEON_NUM_RINGS; ++i) {
-                                if (time_after(rdev->fence_drv[i].last_activity, tmp)) {
-                                        tmp = rdev->fence_drv[i].last_activity;
-                                }
-                        }
-                        /* test if somebody else has already decided that this is a lockup */
-                        if (last_activity != tmp) {
-                                last_activity = tmp;
-                                mutex_unlock(&rdev->ring_lock);
-                                continue;
-                        }
-
-                        if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) {
-                                /* good news we believe it's a lockup */
-                                dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx)\n",
-                                         target_seq[ring]);
-
-                                /* change last activity so nobody else think there is a lockup */
-                                for (i = 0; i < RADEON_NUM_RINGS; ++i) {
-                                        rdev->fence_drv[i].last_activity = jiffies;
-                                }
+        r = radeon_fence_wait_seq(fence->rdev, seq, intr, true);
+        if (r)
+                return r;
 
-                                /* mark the ring as not ready any more */
-                                rdev->ring[ring].ready = false;
-                                mutex_unlock(&rdev->ring_lock);
-                                return -EDEADLK;
-                        }
-                        mutex_unlock(&rdev->ring_lock);
-                }
-        }
+        fence->seq = RADEON_FENCE_SIGNALED_SEQ;
         return 0;
 }
 
@@ -557,7 +442,7 @@ int radeon_fence_wait_any(struct radeon_device *rdev,
                           bool intr)
 {
         uint64_t seq[RADEON_NUM_RINGS];
-        unsigned i;
+        unsigned i, num_rings = 0;
         int r;
 
         for (i = 0; i < RADEON_NUM_RINGS; ++i) {
@@ -567,15 +452,19 @@ int radeon_fence_wait_any(struct radeon_device *rdev,
                         continue;
                 }
 
-                if (fences[i]->seq == RADEON_FENCE_SIGNALED_SEQ) {
-                        /* something was allready signaled */
-                        return 0;
-                }
-
                 seq[i] = fences[i]->seq;
+                ++num_rings;
+
+                /* test if something was allready signaled */
+                if (seq[i] == RADEON_FENCE_SIGNALED_SEQ)
+                        return 0;
         }
 
-        r = radeon_fence_wait_any_seq(rdev, seq, intr);
+        /* nothing to wait for ? */
+        if (num_rings == 0)
+                return -ENOENT;
+
+        r = radeon_fence_wait_seq(rdev, seq, intr, true);
         if (r) {
                 return r;
         }
@@ -594,15 +483,15 @@ int radeon_fence_wait_any(struct radeon_device *rdev,
  */
 int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring)
 {
-        uint64_t seq;
+        uint64_t seq[RADEON_NUM_RINGS] = {};
 
-        seq = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL;
-        if (seq >= rdev->fence_drv[ring].sync_seq[ring]) {
+        seq[ring] = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL;
+        if (seq[ring] >= rdev->fence_drv[ring].sync_seq[ring]) {
                 /* nothing to wait for, last_seq is
                    already the last emited fence */
                 return -ENOENT;
         }
-        return radeon_fence_wait_seq(rdev, seq, ring, false, false);
+        return radeon_fence_wait_seq(rdev, seq, false, false);
 }
 
 /**
@@ -617,14 +506,18 @@ int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring)
  */
 int radeon_fence_wait_empty_locked(struct radeon_device *rdev, int ring)
 {
-        uint64_t seq = rdev->fence_drv[ring].sync_seq[ring];
+        uint64_t seq[RADEON_NUM_RINGS] = {};
         int r;
 
-        r = radeon_fence_wait_seq(rdev, seq, ring, false, false);
+        seq[ring] = rdev->fence_drv[ring].sync_seq[ring];
+        if (!seq[ring])
+                return 0;
+
+        r = radeon_fence_wait_seq(rdev, seq, false, false);
         if (r) {
-                if (r == -EDEADLK) {
+                if (r == -EDEADLK)
                         return -EDEADLK;
-                }
+
                 dev_err(rdev->dev, "error waiting for ring[%d] to become idle (%d)\n",
                         ring, r);
         }
@@ -826,7 +719,6 @@ static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring)
         for (i = 0; i < RADEON_NUM_RINGS; ++i)
                 rdev->fence_drv[ring].sync_seq[i] = 0;
         atomic64_set(&rdev->fence_drv[ring].last_seq, 0);
-        rdev->fence_drv[ring].last_activity = jiffies;
         rdev->fence_drv[ring].initialized = false;
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_gart.c b/drivers/gpu/drm/radeon/radeon_gart.c
index b990b1a2bd50..8a83b89d4709 100644
--- a/drivers/gpu/drm/radeon/radeon_gart.c
+++ b/drivers/gpu/drm/radeon/radeon_gart.c
@@ -607,8 +607,8 @@ static int radeon_vm_evict(struct radeon_device *rdev, struct radeon_vm *vm)
  */
 int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm)
 {
-        unsigned pd_size, pts_size;
-        u64 *pd_addr;
+        unsigned pd_size, pd_entries, pts_size;
+        struct radeon_ib ib;
         int r;
 
         if (vm == NULL) {
@@ -619,8 +619,10 @@ int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm)
                 return 0;
         }
 
-retry:
         pd_size = radeon_vm_directory_size(rdev);
+        pd_entries = radeon_vm_num_pdes(rdev);
+
+retry:
         r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager,
                              &vm->page_directory, pd_size,
                              RADEON_VM_PTB_ALIGN_SIZE, false);
@@ -637,9 +639,31 @@ retry:
         vm->pd_gpu_addr = radeon_sa_bo_gpu_addr(vm->page_directory);
 
         /* Initially clear the page directory */
-        pd_addr = radeon_sa_bo_cpu_addr(vm->page_directory);
-        memset(pd_addr, 0, pd_size);
+        r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib,
+                          NULL, pd_entries * 2 + 64);
+        if (r) {
+                radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence);
+                return r;
+        }
+
+        ib.length_dw = 0;
+
+        radeon_asic_vm_set_page(rdev, &ib, vm->pd_gpu_addr,
+                                0, pd_entries, 0, 0);
+
+        radeon_ib_sync_to(&ib, vm->fence);
+        r = radeon_ib_schedule(rdev, &ib, NULL);
+        if (r) {
+                radeon_ib_free(rdev, &ib);
+                radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence);
+                return r;
+        }
+        radeon_fence_unref(&vm->fence);
+        vm->fence = radeon_fence_ref(ib.fence);
+        radeon_ib_free(rdev, &ib);
+        radeon_fence_unref(&vm->last_flush);
 
+        /* allocate page table array */
         pts_size = radeon_vm_num_pdes(rdev) * sizeof(struct radeon_sa_bo *);
         vm->page_tables = kzalloc(pts_size, GFP_KERNEL);
 
@@ -914,6 +938,26 @@ uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr)
 }
 
 /**
+ * radeon_vm_page_flags - translate page flags to what the hw uses
+ *
+ * @flags: flags comming from userspace
+ *
+ * Translate the flags the userspace ABI uses to hw flags.
+ */
+static uint32_t radeon_vm_page_flags(uint32_t flags)
+{
+        uint32_t hw_flags = 0;
+        hw_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_PTE_VALID : 0;
+        hw_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0;
+        hw_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0;
+        if (flags & RADEON_VM_PAGE_SYSTEM) {
+                hw_flags |= R600_PTE_SYSTEM;
+                hw_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0;
+        }
+        return hw_flags;
+}
+
+/**
  * radeon_vm_update_pdes - make sure that page directory is valid
  *
  * @rdev: radeon_device pointer
@@ -974,7 +1018,11 @@ retry:
                         if (count) {
                                 radeon_asic_vm_set_page(rdev, ib, last_pde,
                                                         last_pt, count, incr,
-                                                        RADEON_VM_PAGE_VALID);
+                                                        R600_PTE_VALID);
+
+                                count *= RADEON_VM_PTE_COUNT;
+                                radeon_asic_vm_set_page(rdev, ib, last_pt, 0,
+                                                        count, 0, 0);
                         }
 
                         count = 1;
@@ -987,8 +1035,11 @@ retry:
 
         if (count) {
                 radeon_asic_vm_set_page(rdev, ib, last_pde, last_pt, count,
-                                        incr, RADEON_VM_PAGE_VALID);
+                                        incr, R600_PTE_VALID);
 
+                count *= RADEON_VM_PTE_COUNT;
+                radeon_asic_vm_set_page(rdev, ib, last_pt, 0,
+                                        count, 0, 0);
         }
 
         return 0;
@@ -1082,7 +1133,6 @@ int radeon_vm_bo_update_pte(struct radeon_device *rdev,
                             struct radeon_bo *bo,
                             struct ttm_mem_reg *mem)
 {
-        unsigned ridx = rdev->asic->vm.pt_ring_index;
         struct radeon_ib ib;
         struct radeon_bo_va *bo_va;
         unsigned nptes, npdes, ndw;
@@ -1151,11 +1201,14 @@ int radeon_vm_bo_update_pte(struct radeon_device *rdev,
         /* reserve space for pde addresses */
         ndw += npdes * 2;
 
+        /* reserve space for clearing new page tables */
+        ndw += npdes * 2 * RADEON_VM_PTE_COUNT;
+
         /* update too big for an IB */
         if (ndw > 0xfffff)
                 return -ENOMEM;
 
-        r = radeon_ib_get(rdev, ridx, &ib, NULL, ndw * 4);
+        r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4);
         ib.length_dw = 0;
 
         r = radeon_vm_update_pdes(rdev, vm, &ib, bo_va->soffset, bo_va->eoffset);
@@ -1165,7 +1218,7 @@ int radeon_vm_bo_update_pte(struct radeon_device *rdev,
         }
 
         radeon_vm_update_ptes(rdev, vm, &ib, bo_va->soffset, bo_va->eoffset,
-                              addr, bo_va->flags);
+                              addr, radeon_vm_page_flags(bo_va->flags));
 
         radeon_ib_sync_to(&ib, vm->fence);
         r = radeon_ib_schedule(rdev, &ib, NULL);
diff --git a/drivers/gpu/drm/radeon/radeon_ioc32.c b/drivers/gpu/drm/radeon/radeon_ioc32.c
index c180df8e84db..bdb0f93e73bc 100644
--- a/drivers/gpu/drm/radeon/radeon_ioc32.c
+++ b/drivers/gpu/drm/radeon/radeon_ioc32.c
@@ -418,7 +418,7 @@ long radeon_kms_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long
         if (nr < DRM_COMMAND_BASE)
                 return drm_compat_ioctl(filp, cmd, arg);
 
-        ret = drm_ioctl(filp, cmd, arg);
+        ret = radeon_drm_ioctl(filp, cmd, arg);
 
         return ret;
 }
diff --git a/drivers/gpu/drm/radeon/radeon_irq_kms.c b/drivers/gpu/drm/radeon/radeon_irq_kms.c
index cc9e8482cf30..ec6240b00469 100644
--- a/drivers/gpu/drm/radeon/radeon_irq_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_irq_kms.c
@@ -32,6 +32,8 @@
 #include "radeon.h"
 #include "atom.h"
 
+#include <linux/pm_runtime.h>
+
 #define RADEON_WAIT_IDLE_TIMEOUT 200
 
 /**
@@ -47,8 +49,12 @@ irqreturn_t radeon_driver_irq_handler_kms(DRM_IRQ_ARGS)
 {
         struct drm_device *dev = (struct drm_device *) arg;
         struct radeon_device *rdev = dev->dev_private;
+        irqreturn_t ret;
 
-        return radeon_irq_process(rdev);
+        ret = radeon_irq_process(rdev);
+        if (ret == IRQ_HANDLED)
+                pm_runtime_mark_last_busy(dev->dev);
+        return ret;
 }
 
 /*
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index d6b36766e8c9..bb8710531a1b 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -32,7 +32,7 @@
 
 #include <linux/vga_switcheroo.h>
 #include <linux/slab.h>
-
+#include <linux/pm_runtime.h>
 /**
  * radeon_driver_unload_kms - Main unload function for KMS.
  *
@@ -50,9 +50,14 @@ int radeon_driver_unload_kms(struct drm_device *dev)
 
         if (rdev == NULL)
                 return 0;
+
         if (rdev->rmmio == NULL)
                 goto done_free;
+
+        pm_runtime_get_sync(dev->dev);
+
         radeon_acpi_fini(rdev);
+        
         radeon_modeset_fini(rdev);
         radeon_device_fini(rdev);
 
@@ -125,9 +130,20 @@ int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags)
                                 "Error during ACPI methods call\n");
         }
 
+        if (radeon_runtime_pm != 0) {
+                pm_runtime_use_autosuspend(dev->dev);
+                pm_runtime_set_autosuspend_delay(dev->dev, 5000);
+                pm_runtime_set_active(dev->dev);
+                pm_runtime_allow(dev->dev);
+                pm_runtime_mark_last_busy(dev->dev);
+                pm_runtime_put_autosuspend(dev->dev);
+        }
+
 out:
         if (r)
                 radeon_driver_unload_kms(dev);
+
+
         return r;
 }
 
@@ -475,9 +491,14 @@ void radeon_driver_lastclose_kms(struct drm_device *dev)
 int radeon_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 {
         struct radeon_device *rdev = dev->dev_private;
+        int r;
 
         file_priv->driver_priv = NULL;
 
+        r = pm_runtime_get_sync(dev->dev);
+        if (r < 0)
+                return r;
+
         /* new gpu have virtual address space support */
         if (rdev->family >= CHIP_CAYMAN) {
                 struct radeon_fpriv *fpriv;
@@ -506,6 +527,9 @@ int radeon_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
 
                 file_priv->driver_priv = fpriv;
         }
+
+        pm_runtime_mark_last_busy(dev->dev);
+        pm_runtime_put_autosuspend(dev->dev);
         return 0;
 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
index 7cb178a34a0f..0c7b8c66301b 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
@@ -1056,6 +1056,26 @@ static void radeon_crtc_commit(struct drm_crtc *crtc)
         }
 }
 
+static void radeon_crtc_disable(struct drm_crtc *crtc)
+{
+        radeon_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
+        if (crtc->fb) {
+                int r;
+                struct radeon_framebuffer *radeon_fb;
+                struct radeon_bo *rbo;
+
+                radeon_fb = to_radeon_framebuffer(crtc->fb);
+                rbo = gem_to_radeon_bo(radeon_fb->obj);
+                r = radeon_bo_reserve(rbo, false);
+                if (unlikely(r))
+                        DRM_ERROR("failed to reserve rbo before unpin\n");
+                else {
+                        radeon_bo_unpin(rbo);
+                        radeon_bo_unreserve(rbo);
+                }
+        }
+}
+
 static const struct drm_crtc_helper_funcs legacy_helper_funcs = {
         .dpms = radeon_crtc_dpms,
         .mode_fixup = radeon_crtc_mode_fixup,
@@ -1065,6 +1085,7 @@ static const struct drm_crtc_helper_funcs legacy_helper_funcs = {
         .prepare = radeon_crtc_prepare,
         .commit = radeon_crtc_commit,
         .load_lut = radeon_crtc_load_lut,
+        .disable = radeon_crtc_disable
 };
 
 
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_encoders.c b/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
index 62cd512f5c8d..c89971d904c3 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_encoders.c
@@ -392,7 +392,7 @@ void radeon_legacy_backlight_init(struct radeon_encoder *radeon_encoder,
         props.type = BACKLIGHT_RAW;
         snprintf(bl_name, sizeof(bl_name),
                  "radeon_bl%d", dev->primary->index);
-        bd = backlight_device_register(bl_name, &drm_connector->kdev,
+        bd = backlight_device_register(bl_name, drm_connector->kdev,
                                        pdata, &radeon_backlight_ops, &props);
         if (IS_ERR(bd)) {
                 DRM_ERROR("Backlight registration failed\n");
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index ef63d3f00b2f..3f0dd664af90 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -249,6 +249,8 @@ struct radeon_mode_info {
         struct drm_property *underscan_vborder_property;
         /* audio */
         struct drm_property *audio_property;
+        /* FMT dithering */
+        struct drm_property *dither_property;
         /* hardcoded DFP edid from BIOS */
         struct edid *bios_hardcoded_edid;
         int bios_hardcoded_edid_size;
@@ -479,6 +481,11 @@ enum radeon_connector_audio {
         RADEON_AUDIO_AUTO = 2
 };
 
+enum radeon_connector_dither {
+        RADEON_FMT_DITHER_DISABLE = 0,
+        RADEON_FMT_DITHER_ENABLE = 1,
+};
+
 struct radeon_connector {
         struct drm_connector base;
         uint32_t connector_id;
@@ -498,6 +505,7 @@ struct radeon_connector {
         struct radeon_router router;
         struct radeon_i2c_chan *router_bus;
         enum radeon_connector_audio audio;
+        enum radeon_connector_dither dither;
 };
 
 struct radeon_framebuffer {
@@ -758,7 +766,8 @@ extern int radeon_crtc_cursor_move(struct drm_crtc *crtc,
                                    int x, int y);
 
 extern int radeon_get_crtc_scanoutpos(struct drm_device *dev, int crtc,
-                                      int *vpos, int *hpos);
+                                      int *vpos, int *hpos, ktime_t *stime,
+                                      ktime_t *etime);
 
 extern bool radeon_combios_check_hardcoded_edid(struct radeon_device *rdev);
 extern struct edid *
@@ -850,6 +859,12 @@ void radeon_legacy_tv_mode_set(struct drm_encoder *encoder,
                                struct drm_display_mode *mode,
                                struct drm_display_mode *adjusted_mode);
 
+/* fmt blocks */
+void avivo_program_fmt(struct drm_encoder *encoder);
+void dce3_program_fmt(struct drm_encoder *encoder);
+void dce4_program_fmt(struct drm_encoder *encoder);
+void dce8_program_fmt(struct drm_encoder *encoder);
+
 /* fbdev layer */
 int radeon_fbdev_init(struct radeon_device *rdev);
 void radeon_fbdev_fini(struct radeon_device *rdev);
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index 4f6b7fc7ad3c..866ace070b91 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -508,17 +508,21 @@ static ssize_t radeon_set_dpm_forced_performance_level(struct device *dev,
         } else if (strncmp("auto", buf, strlen("auto")) == 0) {
                 level = RADEON_DPM_FORCED_LEVEL_AUTO;
         } else {
-                mutex_unlock(&rdev->pm.mutex);
                 count = -EINVAL;
                 goto fail;
         }
         if (rdev->asic->dpm.force_performance_level) {
+                if (rdev->pm.dpm.thermal_active) {
+                        count = -EINVAL;
+                        goto fail;
+                }
                 ret = radeon_dpm_force_performance_level(rdev, level);
                 if (ret)
                         count = -EINVAL;
         }
-        mutex_unlock(&rdev->pm.mutex);
 fail:
+        mutex_unlock(&rdev->pm.mutex);
+
         return count;
 }
 
@@ -881,11 +885,12 @@ static void radeon_dpm_change_power_state_locked(struct radeon_device *rdev)
                 }
         }
 
-        printk("switching from power state:\n");
-        radeon_dpm_print_power_state(rdev, rdev->pm.dpm.current_ps);
-        printk("switching to power state:\n");
-        radeon_dpm_print_power_state(rdev, rdev->pm.dpm.requested_ps);
-
+        if (radeon_dpm == 1) {
+                printk("switching from power state:\n");
+                radeon_dpm_print_power_state(rdev, rdev->pm.dpm.current_ps);
+                printk("switching to power state:\n");
+                radeon_dpm_print_power_state(rdev, rdev->pm.dpm.requested_ps);
+        }
         mutex_lock(&rdev->ddev->struct_mutex);
         down_write(&rdev->pm.mclk_lock);
         mutex_lock(&rdev->ring_lock);
@@ -918,12 +923,16 @@ static void radeon_dpm_change_power_state_locked(struct radeon_device *rdev)
         radeon_dpm_post_set_power_state(rdev);
 
         if (rdev->asic->dpm.force_performance_level) {
-                if (rdev->pm.dpm.thermal_active)
+                if (rdev->pm.dpm.thermal_active) {
+                        enum radeon_dpm_forced_level level = rdev->pm.dpm.forced_level;
                         /* force low perf level for thermal */
                         radeon_dpm_force_performance_level(rdev, RADEON_DPM_FORCED_LEVEL_LOW);
-                else
-                        /* otherwise, enable auto */
-                        radeon_dpm_force_performance_level(rdev, RADEON_DPM_FORCED_LEVEL_AUTO);
+                        /* save the user's level */
+                        rdev->pm.dpm.forced_level = level;
+                } else {
+                        /* otherwise, user selected level */
+                        radeon_dpm_force_performance_level(rdev, rdev->pm.dpm.forced_level);
+                }
         }
 
 done:
@@ -1179,7 +1188,8 @@ static int radeon_pm_init_dpm(struct radeon_device *rdev)
         mutex_lock(&rdev->pm.mutex);
         radeon_dpm_init(rdev);
         rdev->pm.dpm.current_ps = rdev->pm.dpm.requested_ps = rdev->pm.dpm.boot_ps;
-        radeon_dpm_print_power_states(rdev);
+        if (radeon_dpm == 1)
+                radeon_dpm_print_power_states(rdev);
         radeon_dpm_setup_asic(rdev);
         ret = radeon_dpm_enable(rdev);
         mutex_unlock(&rdev->pm.mutex);
@@ -1241,6 +1251,24 @@ int radeon_pm_init(struct radeon_device *rdev)
         case CHIP_RV670:
         case CHIP_RS780:
         case CHIP_RS880:
+        case CHIP_CAYMAN:
+        case CHIP_ARUBA:
+        case CHIP_BONAIRE:
+        case CHIP_KABINI:
+        case CHIP_KAVERI:
+        case CHIP_HAWAII:
+                /* DPM requires the RLC, RV770+ dGPU requires SMC */
+                if (!rdev->rlc_fw)
+                        rdev->pm.pm_method = PM_METHOD_PROFILE;
+                else if ((rdev->family >= CHIP_RV770) &&
+                         (!(rdev->flags & RADEON_IS_IGP)) &&
+                         (!rdev->smc_fw))
+                        rdev->pm.pm_method = PM_METHOD_PROFILE;
+                else if (radeon_dpm == 1)
+                        rdev->pm.pm_method = PM_METHOD_DPM;
+                else
+                        rdev->pm.pm_method = PM_METHOD_PROFILE;
+                break;
         case CHIP_RV770:
         case CHIP_RV730:
         case CHIP_RV710:
@@ -1256,16 +1284,11 @@ int radeon_pm_init(struct radeon_device *rdev)
         case CHIP_BARTS:
         case CHIP_TURKS:
         case CHIP_CAICOS:
-        case CHIP_CAYMAN:
-        case CHIP_ARUBA:
         case CHIP_TAHITI:
         case CHIP_PITCAIRN:
         case CHIP_VERDE:
         case CHIP_OLAND:
         case CHIP_HAINAN:
-        case CHIP_BONAIRE:
-        case CHIP_KABINI:
-        case CHIP_KAVERI:
                 /* DPM requires the RLC, RV770+ dGPU requires SMC */
                 if (!rdev->rlc_fw)
                         rdev->pm.pm_method = PM_METHOD_PROFILE;
@@ -1273,10 +1296,10 @@ int radeon_pm_init(struct radeon_device *rdev)
                          (!(rdev->flags & RADEON_IS_IGP)) &&
                          (!rdev->smc_fw))
                         rdev->pm.pm_method = PM_METHOD_PROFILE;
-                else if (radeon_dpm == 1)
-                        rdev->pm.pm_method = PM_METHOD_DPM;
-                else
+                else if (radeon_dpm == 0)
                         rdev->pm.pm_method = PM_METHOD_PROFILE;
+                else
+                        rdev->pm.pm_method = PM_METHOD_DPM;
                 break;
         default:
                 /* default to profile method */
@@ -1468,7 +1491,7 @@ static bool radeon_pm_in_vbl(struct radeon_device *rdev)
          */
         for (crtc = 0; (crtc < rdev->num_crtc) && in_vbl; crtc++) {
                 if (rdev->pm.active_crtcs & (1 << crtc)) {
-                        vbl_status = radeon_get_crtc_scanoutpos(rdev->ddev, crtc, &vpos, &hpos);
+                        vbl_status = radeon_get_crtc_scanoutpos(rdev->ddev, crtc, &vpos, &hpos, NULL, NULL);
                         if ((vbl_status & DRM_SCANOUTPOS_VALID) &&
                             !(vbl_status & DRM_SCANOUTPOS_INVBL))
                                 in_vbl = false;
diff --git a/drivers/gpu/drm/radeon/radeon_trace.h b/drivers/gpu/drm/radeon/radeon_trace.h
index f7e367815964..811bca691b36 100644
--- a/drivers/gpu/drm/radeon/radeon_trace.h
+++ b/drivers/gpu/drm/radeon/radeon_trace.h
@@ -47,6 +47,30 @@ TRACE_EVENT(radeon_cs,
                       __entry->fences)
 );
 
+TRACE_EVENT(radeon_vm_set_page,
+            TP_PROTO(uint64_t pe, uint64_t addr, unsigned count,
+                     uint32_t incr, uint32_t flags),
+            TP_ARGS(pe, addr, count, incr, flags),
+            TP_STRUCT__entry(
+                             __field(u64, pe)
+                             __field(u64, addr)
+                             __field(u32, count)
+                             __field(u32, incr)
+                             __field(u32, flags)
+                             ),
+
+            TP_fast_assign(
+                           __entry->pe = pe;
+                           __entry->addr = addr;
+                           __entry->count = count;
+                           __entry->incr = incr;
+                           __entry->flags = flags;
+                           ),
+            TP_printk("pe=%010Lx, addr=%010Lx, incr=%u, flags=%08x, count=%u",
+                      __entry->pe, __entry->addr, __entry->incr,
+                      __entry->flags, __entry->count)
+);
+
 DECLARE_EVENT_CLASS(radeon_fence_request,
 
             TP_PROTO(struct drm_device *dev, u32 seqno),
diff --git a/drivers/gpu/drm/radeon/radeon_ucode.h b/drivers/gpu/drm/radeon/radeon_ucode.h
index 33858364fe89..a77cd274dfc3 100644
--- a/drivers/gpu/drm/radeon/radeon_ucode.h
+++ b/drivers/gpu/drm/radeon/radeon_ucode.h
@@ -59,6 +59,7 @@
 #define SI_MC_UCODE_SIZE             7769
 #define OLAND_MC_UCODE_SIZE          7863
 #define CIK_MC_UCODE_SIZE            7866
+#define HAWAII_MC_UCODE_SIZE         7933
 
 /* SDMA */
 #define CIK_SDMA_UCODE_SIZE          1050
@@ -143,4 +144,7 @@
 #define BONAIRE_SMC_UCODE_START      0x20000
 #define BONAIRE_SMC_UCODE_SIZE       0x1FDEC
 
+#define HAWAII_SMC_UCODE_START       0x20000
+#define HAWAII_SMC_UCODE_SIZE        0x1FDEC
+
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon_uvd.c b/drivers/gpu/drm/radeon/radeon_uvd.c
index 308eff5be1b4..373d088bac66 100644
--- a/drivers/gpu/drm/radeon/radeon_uvd.c
+++ b/drivers/gpu/drm/radeon/radeon_uvd.c
@@ -97,6 +97,7 @@ int radeon_uvd_init(struct radeon_device *rdev)
         case CHIP_BONAIRE:
         case CHIP_KABINI:
         case CHIP_KAVERI:
+        case CHIP_HAWAII:
                 fw_name = FIRMWARE_BONAIRE;
                 break;
 
@@ -240,6 +241,8 @@ void radeon_uvd_free_handles(struct radeon_device *rdev, struct drm_file *filp)
                 if (handle != 0 && rdev->uvd.filp[i] == filp) {
                         struct radeon_fence *fence;
 
+                        radeon_uvd_note_usage(rdev);
+
                         r = radeon_uvd_get_destroy_msg(rdev,
                                 R600_RING_TYPE_UVD_INDEX, handle, &fence);
                         if (r) {
@@ -620,7 +623,7 @@ static int radeon_uvd_send_msg(struct radeon_device *rdev,
         if (r) 
                 goto err;
 
-        r = radeon_ib_get(rdev, ring, &ib, NULL, 16);
+        r = radeon_ib_get(rdev, ring, &ib, NULL, 64);
         if (r)
                 goto err;
 
diff --git a/drivers/gpu/drm/radeon/rs600.c b/drivers/gpu/drm/radeon/rs600.c
index 6acba8017b9a..76cc8d3aafec 100644
--- a/drivers/gpu/drm/radeon/rs600.c
+++ b/drivers/gpu/drm/radeon/rs600.c
@@ -153,6 +153,70 @@ u32 rs600_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base)
         return RREG32(AVIVO_D1GRPH_UPDATE + radeon_crtc->crtc_offset) & AVIVO_D1GRPH_SURFACE_UPDATE_PENDING;
 }
 
+void avivo_program_fmt(struct drm_encoder *encoder)
+{
+        struct drm_device *dev = encoder->dev;
+        struct radeon_device *rdev = dev->dev_private;
+        struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
+        struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
+        int bpc = 0;
+        u32 tmp = 0;
+        enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
+
+        if (connector) {
+                struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+                bpc = radeon_get_monitor_bpc(connector);
+                dither = radeon_connector->dither;
+        }
+
+        /* LVDS FMT is set up by atom */
+        if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
+                return;
+
+        if (bpc == 0)
+                return;
+
+        switch (bpc) {
+        case 6:
+                if (dither == RADEON_FMT_DITHER_ENABLE)
+                        /* XXX sort out optimal dither settings */
+                        tmp |= AVIVO_TMDS_BIT_DEPTH_CONTROL_SPATIAL_DITHER_EN;
+                else
+                        tmp |= AVIVO_TMDS_BIT_DEPTH_CONTROL_TRUNCATE_EN;
+                break;
+        case 8:
+                if (dither == RADEON_FMT_DITHER_ENABLE)
+                        /* XXX sort out optimal dither settings */
+                        tmp |= (AVIVO_TMDS_BIT_DEPTH_CONTROL_SPATIAL_DITHER_EN |
+                                AVIVO_TMDS_BIT_DEPTH_CONTROL_SPATIAL_DITHER_DEPTH);
+                else
+                        tmp |= (AVIVO_TMDS_BIT_DEPTH_CONTROL_TRUNCATE_EN |
+                                AVIVO_TMDS_BIT_DEPTH_CONTROL_TRUNCATE_DEPTH);
+                break;
+        case 10:
+        default:
+                /* not needed */
+                break;
+        }
+
+        switch (radeon_encoder->encoder_id) {
+        case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_TMDS1:
+                WREG32(AVIVO_TMDSA_BIT_DEPTH_CONTROL, tmp);
+                break;
+        case ENCODER_OBJECT_ID_INTERNAL_LVTM1:
+                WREG32(AVIVO_LVTMA_BIT_DEPTH_CONTROL, tmp);
+                break;
+        case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DVO1:
+                WREG32(AVIVO_DVOA_BIT_DEPTH_CONTROL, tmp);
+                break;
+        case ENCODER_OBJECT_ID_INTERNAL_DDI:
+                WREG32(AVIVO_DDIA_BIT_DEPTH_CONTROL, tmp);
+                break;
+        default:
+                break;
+        }
+}
+
 void rs600_pm_misc(struct radeon_device *rdev)
 {
         int requested_index = rdev->pm.requested_power_state_index;
diff --git a/drivers/gpu/drm/radeon/rs690.c b/drivers/gpu/drm/radeon/rs690.c
index 1447d794c22a..1c560629575a 100644
--- a/drivers/gpu/drm/radeon/rs690.c
+++ b/drivers/gpu/drm/radeon/rs690.c
@@ -345,9 +345,11 @@ static void rs690_crtc_bandwidth_compute(struct radeon_device *rdev,
                 if (max_bandwidth.full > rdev->pm.sideport_bandwidth.full &&
                         rdev->pm.sideport_bandwidth.full)
                         max_bandwidth = rdev->pm.sideport_bandwidth;
-                read_delay_latency.full = dfixed_const(370 * 800 * 1000);
-                read_delay_latency.full = dfixed_div(read_delay_latency,
-                        rdev->pm.igp_sideport_mclk);
+                read_delay_latency.full = dfixed_const(370 * 800);
+                a.full = dfixed_const(1000);
+                b.full = dfixed_div(rdev->pm.igp_sideport_mclk, a);
+                read_delay_latency.full = dfixed_div(read_delay_latency, b);
+                read_delay_latency.full = dfixed_mul(read_delay_latency, a);
         } else {
                 if (max_bandwidth.full > rdev->pm.k8_bandwidth.full &&
                         rdev->pm.k8_bandwidth.full)
@@ -488,14 +490,10 @@ static void rs690_compute_mode_priority(struct radeon_device *rdev,
                 }
                 if (wm0->priority_mark.full > priority_mark02.full)
                         priority_mark02.full = wm0->priority_mark.full;
-                if (dfixed_trunc(priority_mark02) < 0)
-                        priority_mark02.full = 0;
                 if (wm0->priority_mark_max.full > priority_mark02.full)
                         priority_mark02.full = wm0->priority_mark_max.full;
                 if (wm1->priority_mark.full > priority_mark12.full)
                         priority_mark12.full = wm1->priority_mark.full;
-                if (dfixed_trunc(priority_mark12) < 0)
-                        priority_mark12.full = 0;
                 if (wm1->priority_mark_max.full > priority_mark12.full)
                         priority_mark12.full = wm1->priority_mark_max.full;
                 *d1mode_priority_a_cnt = dfixed_trunc(priority_mark02);
@@ -526,8 +524,6 @@ static void rs690_compute_mode_priority(struct radeon_device *rdev,
                 }
                 if (wm0->priority_mark.full > priority_mark02.full)
                         priority_mark02.full = wm0->priority_mark.full;
-                if (dfixed_trunc(priority_mark02) < 0)
-                        priority_mark02.full = 0;
                 if (wm0->priority_mark_max.full > priority_mark02.full)
                         priority_mark02.full = wm0->priority_mark_max.full;
                 *d1mode_priority_a_cnt = dfixed_trunc(priority_mark02);
@@ -555,8 +551,6 @@ static void rs690_compute_mode_priority(struct radeon_device *rdev,
                 }
                 if (wm1->priority_mark.full > priority_mark12.full)
                         priority_mark12.full = wm1->priority_mark.full;
-                if (dfixed_trunc(priority_mark12) < 0)
-                        priority_mark12.full = 0;
                 if (wm1->priority_mark_max.full > priority_mark12.full)
                         priority_mark12.full = wm1->priority_mark_max.full;
                 *d2mode_priority_a_cnt = dfixed_trunc(priority_mark12);
diff --git a/drivers/gpu/drm/radeon/rv515.c b/drivers/gpu/drm/radeon/rv515.c
index 873eb4b193b4..5d1c316115ef 100644
--- a/drivers/gpu/drm/radeon/rv515.c
+++ b/drivers/gpu/drm/radeon/rv515.c
@@ -1155,14 +1155,10 @@ static void rv515_compute_mode_priority(struct radeon_device *rdev,
                 }
                 if (wm0->priority_mark.full > priority_mark02.full)
                         priority_mark02.full = wm0->priority_mark.full;
-                if (dfixed_trunc(priority_mark02) < 0)
-                        priority_mark02.full = 0;
                 if (wm0->priority_mark_max.full > priority_mark02.full)
                         priority_mark02.full = wm0->priority_mark_max.full;
                 if (wm1->priority_mark.full > priority_mark12.full)
                         priority_mark12.full = wm1->priority_mark.full;
-                if (dfixed_trunc(priority_mark12) < 0)
-                        priority_mark12.full = 0;
                 if (wm1->priority_mark_max.full > priority_mark12.full)
                         priority_mark12.full = wm1->priority_mark_max.full;
                 *d1mode_priority_a_cnt = dfixed_trunc(priority_mark02);
@@ -1193,8 +1189,6 @@ static void rv515_compute_mode_priority(struct radeon_device *rdev,
                 }
                 if (wm0->priority_mark.full > priority_mark02.full)
                         priority_mark02.full = wm0->priority_mark.full;
-                if (dfixed_trunc(priority_mark02) < 0)
-                        priority_mark02.full = 0;
                 if (wm0->priority_mark_max.full > priority_mark02.full)
                         priority_mark02.full = wm0->priority_mark_max.full;
                 *d1mode_priority_a_cnt = dfixed_trunc(priority_mark02);
@@ -1222,8 +1216,6 @@ static void rv515_compute_mode_priority(struct radeon_device *rdev,
                 }
                 if (wm1->priority_mark.full > priority_mark12.full)
                         priority_mark12.full = wm1->priority_mark.full;
-                if (dfixed_trunc(priority_mark12) < 0)
-                        priority_mark12.full = 0;
                 if (wm1->priority_mark_max.full > priority_mark12.full)
                         priority_mark12.full = wm1->priority_mark_max.full;
                 *d2mode_priority_a_cnt = dfixed_trunc(priority_mark12);
diff --git a/drivers/gpu/drm/radeon/rv6xx_dpm.c b/drivers/gpu/drm/radeon/rv6xx_dpm.c
index 5811d277a36a..26633a025252 100644
--- a/drivers/gpu/drm/radeon/rv6xx_dpm.c
+++ b/drivers/gpu/drm/radeon/rv6xx_dpm.c
@@ -407,9 +407,9 @@ static void rv6xx_enable_engine_feedback_and_reference_sync(struct radeon_device
         WREG32_P(SPLL_CNTL_MODE, SPLL_DIV_SYNC, ~SPLL_DIV_SYNC);
 }
 
-static u64 rv6xx_clocks_per_unit(u32 unit)
+static u32 rv6xx_clocks_per_unit(u32 unit)
 {
-        u64 tmp = 1 << (2 * unit);
+        u32 tmp = 1 << (2 * unit);
 
         return tmp;
 }
@@ -417,7 +417,7 @@ static u64 rv6xx_clocks_per_unit(u32 unit)
 static u32 rv6xx_scale_count_given_unit(struct radeon_device *rdev,
                                         u32 unscaled_count, u32 unit)
 {
-        u32 count_per_unit = (u32)rv6xx_clocks_per_unit(unit);
+        u32 count_per_unit = rv6xx_clocks_per_unit(unit);
 
         return (unscaled_count + count_per_unit - 1) / count_per_unit;
 }
diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c
index d96f7cbca0a1..6a64ccaa0695 100644
--- a/drivers/gpu/drm/radeon/si.c
+++ b/drivers/gpu/drm/radeon/si.c
@@ -78,11 +78,6 @@ extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_
 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
-extern void si_dma_vm_set_page(struct radeon_device *rdev,
-                               struct radeon_ib *ib,
-                               uint64_t pe,
-                               uint64_t addr, unsigned count,
-                               uint32_t incr, uint32_t flags);
 static void si_enable_gui_idle_interrupt(struct radeon_device *rdev,
                                          bool enable);
 static void si_fini_pg(struct radeon_device *rdev);
@@ -4673,61 +4668,6 @@ static void si_vm_decode_fault(struct radeon_device *rdev,
                block, mc_id);
 }
 
-/**
- * si_vm_set_page - update the page tables using the CP
- *
- * @rdev: radeon_device pointer
- * @ib: indirect buffer to fill with commands
- * @pe: addr of the page entry
- * @addr: dst addr to write into pe
- * @count: number of page entries to update
- * @incr: increase next addr by incr bytes
- * @flags: access flags
- *
- * Update the page tables using the CP (SI).
- */
-void si_vm_set_page(struct radeon_device *rdev,
-                    struct radeon_ib *ib,
-                    uint64_t pe,
-                    uint64_t addr, unsigned count,
-                    uint32_t incr, uint32_t flags)
-{
-        uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
-        uint64_t value;
-        unsigned ndw;
-
-        if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
-                while (count) {
-                        ndw = 2 + count * 2;
-                        if (ndw > 0x3FFE)
-                                ndw = 0x3FFE;
-
-                        ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
-                        ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
-                                        WRITE_DATA_DST_SEL(1));
-                        ib->ptr[ib->length_dw++] = pe;
-                        ib->ptr[ib->length_dw++] = upper_32_bits(pe);
-                        for (; ndw > 2; ndw -= 2, --count, pe += 8) {
-                                if (flags & RADEON_VM_PAGE_SYSTEM) {
-                                        value = radeon_vm_map_gart(rdev, addr);
-                                        value &= 0xFFFFFFFFFFFFF000ULL;
-                                } else if (flags & RADEON_VM_PAGE_VALID) {
-                                        value = addr;
-                                } else {
-                                        value = 0;
-                                }
-                                addr += incr;
-                                value |= r600_flags;
-                                ib->ptr[ib->length_dw++] = value;
-                                ib->ptr[ib->length_dw++] = upper_32_bits(value);
-                        }
-                }
-        } else {
-                /* DMA */
-                si_dma_vm_set_page(rdev, ib, pe, addr, count, incr, flags);
-        }
-}
-
 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
 {
         struct radeon_ring *ring = &rdev->ring[ridx];
@@ -5372,52 +5312,53 @@ void si_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
         if (buffer == NULL)
                 return;
 
-        buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
-        buffer[count++] = PACKET3_PREAMBLE_BEGIN_CLEAR_STATE;
+        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+        buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
 
-        buffer[count++] = PACKET3(PACKET3_CONTEXT_CONTROL, 1);
-        buffer[count++] = 0x80000000;
-        buffer[count++] = 0x80000000;
+        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CONTEXT_CONTROL, 1));
+        buffer[count++] = cpu_to_le32(0x80000000);
+        buffer[count++] = cpu_to_le32(0x80000000);
 
         for (sect = rdev->rlc.cs_data; sect->section != NULL; ++sect) {
                 for (ext = sect->section; ext->extent != NULL; ++ext) {
                         if (sect->id == SECT_CONTEXT) {
-                                buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count);
-                                buffer[count++] = ext->reg_index - 0xa000;
+                                buffer[count++] =
+                                        cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, ext->reg_count));
+                                buffer[count++] = cpu_to_le32(ext->reg_index - 0xa000);
                                 for (i = 0; i < ext->reg_count; i++)
-                                        buffer[count++] = ext->extent[i];
+                                        buffer[count++] = cpu_to_le32(ext->extent[i]);
                         } else {
                                 return;
                         }
                 }
         }
 
-        buffer[count++] = PACKET3(PACKET3_SET_CONTEXT_REG, 1);
-        buffer[count++] = PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START;
+        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_SET_CONTEXT_REG, 1));
+        buffer[count++] = cpu_to_le32(PA_SC_RASTER_CONFIG - PACKET3_SET_CONTEXT_REG_START);
         switch (rdev->family) {
         case CHIP_TAHITI:
         case CHIP_PITCAIRN:
-                buffer[count++] = 0x2a00126a;
+                buffer[count++] = cpu_to_le32(0x2a00126a);
                 break;
         case CHIP_VERDE:
-                buffer[count++] = 0x0000124a;
+                buffer[count++] = cpu_to_le32(0x0000124a);
                 break;
         case CHIP_OLAND:
-                buffer[count++] = 0x00000082;
+                buffer[count++] = cpu_to_le32(0x00000082);
                 break;
         case CHIP_HAINAN:
-                buffer[count++] = 0x00000000;
+                buffer[count++] = cpu_to_le32(0x00000000);
                 break;
         default:
-                buffer[count++] = 0x00000000;
+                buffer[count++] = cpu_to_le32(0x00000000);
                 break;
         }
 
-        buffer[count++] = PACKET3(PACKET3_PREAMBLE_CNTL, 0);
-        buffer[count++] = PACKET3_PREAMBLE_END_CLEAR_STATE;
+        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_PREAMBLE_CNTL, 0));
+        buffer[count++] = cpu_to_le32(PACKET3_PREAMBLE_END_CLEAR_STATE);
 
-        buffer[count++] = PACKET3(PACKET3_CLEAR_STATE, 0);
-        buffer[count++] = 0;
+        buffer[count++] = cpu_to_le32(PACKET3(PACKET3_CLEAR_STATE, 0));
+        buffer[count++] = cpu_to_le32(0);
 }
 
 static void si_init_pg(struct radeon_device *rdev)
diff --git a/drivers/gpu/drm/radeon/si_dma.c b/drivers/gpu/drm/radeon/si_dma.c
index 49909d23dfce..8e8f46133532 100644
--- a/drivers/gpu/drm/radeon/si_dma.c
+++ b/drivers/gpu/drm/radeon/si_dma.c
@@ -24,6 +24,7 @@
 #include <drm/drmP.h>
 #include "radeon.h"
 #include "radeon_asic.h"
+#include "radeon_trace.h"
 #include "sid.h"
 
 u32 si_gpu_check_soft_reset(struct radeon_device *rdev);
@@ -75,11 +76,12 @@ void si_dma_vm_set_page(struct radeon_device *rdev,
                         uint64_t addr, unsigned count,
                         uint32_t incr, uint32_t flags)
 {
-        uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
         uint64_t value;
         unsigned ndw;
 
-        if (flags & RADEON_VM_PAGE_SYSTEM) {
+        trace_radeon_vm_set_page(pe, addr, count, incr, flags);
+
+        if (flags & R600_PTE_SYSTEM) {
                 while (count) {
                         ndw = count * 2;
                         if (ndw > 0xFFFFE)
@@ -90,16 +92,10 @@ void si_dma_vm_set_page(struct radeon_device *rdev,
                         ib->ptr[ib->length_dw++] = pe;
                         ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
                         for (; ndw > 0; ndw -= 2, --count, pe += 8) {
-                                if (flags & RADEON_VM_PAGE_SYSTEM) {
-                                        value = radeon_vm_map_gart(rdev, addr);
-                                        value &= 0xFFFFFFFFFFFFF000ULL;
-                                } else if (flags & RADEON_VM_PAGE_VALID) {
-                                        value = addr;
-                                } else {
-                                        value = 0;
-                                }
+                                value = radeon_vm_map_gart(rdev, addr);
+                                value &= 0xFFFFFFFFFFFFF000ULL;
                                 addr += incr;
-                                value |= r600_flags;
+                                value |= flags;
                                 ib->ptr[ib->length_dw++] = value;
                                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
                         }
@@ -110,7 +106,7 @@ void si_dma_vm_set_page(struct radeon_device *rdev,
                         if (ndw > 0xFFFFE)
                                 ndw = 0xFFFFE;
 
-                        if (flags & RADEON_VM_PAGE_VALID)
+                        if (flags & R600_PTE_VALID)
                                 value = addr;
                         else
                                 value = 0;
@@ -118,7 +114,7 @@ void si_dma_vm_set_page(struct radeon_device *rdev,
                         ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
                         ib->ptr[ib->length_dw++] = pe; /* dst addr */
                         ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
-                        ib->ptr[ib->length_dw++] = r600_flags; /* mask */
+                        ib->ptr[ib->length_dw++] = flags; /* mask */
                         ib->ptr[ib->length_dw++] = 0;
                         ib->ptr[ib->length_dw++] = value; /* value */
                         ib->ptr[ib->length_dw++] = upper_32_bits(value);
diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c
index 2332aa1bf93c..0b00c790fb77 100644
--- a/drivers/gpu/drm/radeon/si_dpm.c
+++ b/drivers/gpu/drm/radeon/si_dpm.c
@@ -3589,7 +3589,12 @@ static void si_program_display_gap(struct radeon_device *rdev)
                 WREG32(DCCG_DISP_SLOW_SELECT_REG, tmp);
         }
 
-        si_notify_smc_display_change(rdev, rdev->pm.dpm.new_active_crtc_count > 0);
+        /* Setting this to false forces the performance state to low if the crtcs are disabled.
+         * This can be a problem on PowerXpress systems or if you want to use the card
+         * for offscreen rendering or compute if there are no crtcs enabled.  Set it to
+         * true for now so that performance scales even if the displays are off.
+         */
+        si_notify_smc_display_change(rdev, true /*rdev->pm.dpm.new_active_crtc_count > 0*/);
 }
 
 static void si_enable_spread_spectrum(struct radeon_device *rdev, bool enable)
@@ -4553,7 +4558,7 @@ static int si_init_smc_table(struct radeon_device *rdev)
                 table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
 
         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REVERT_GPIO5_POLARITY)
-                table->systemFlags |= PPSMC_EXTRAFLAGS_AC2DC_GPIO5_POLARITY_HIGH;
+                table->extraFlags |= PPSMC_EXTRAFLAGS_AC2DC_GPIO5_POLARITY_HIGH;
 
         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_VRHOT_GPIO_CONFIGURABLE) {
                 table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT_PROG_GPIO;
diff --git a/drivers/gpu/drm/radeon/sid.h b/drivers/gpu/drm/radeon/sid.h
index 7e2e0ea66a00..b322acc48097 100644
--- a/drivers/gpu/drm/radeon/sid.h
+++ b/drivers/gpu/drm/radeon/sid.h
@@ -478,7 +478,7 @@
 #define         STATE3_MASK                             (0x1f << 15)
 #define         STATE3_SHIFT                            15
 
-#define MC_SEQ_TRAIN_WAKEUP_CNTL                        0x2808
+#define MC_SEQ_TRAIN_WAKEUP_CNTL                        0x28e8
 #define         TRAIN_DONE_D0                           (1 << 30)
 #define         TRAIN_DONE_D1                           (1 << 31)
 
@@ -683,6 +683,51 @@
  * bit5 = 176.4 kHz
  * bit6 = 192 kHz
  */
+
+#define AZ_F0_CODEC_PIN_CONTROL_RESPONSE_LIPSYNC         0x37
+#       define VIDEO_LIPSYNC(x)                           (((x) & 0xff) << 0)
+#       define AUDIO_LIPSYNC(x)                           (((x) & 0xff) << 8)
+/* VIDEO_LIPSYNC, AUDIO_LIPSYNC
+ * 0   = invalid
+ * x   = legal delay value
+ * 255 = sync not supported
+ */
+#define AZ_F0_CODEC_PIN_CONTROL_RESPONSE_HBR             0x38
+#       define HBR_CAPABLE                                (1 << 0) /* enabled by default */
+
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO0               0x3a
+#       define MANUFACTURER_ID(x)                        (((x) & 0xffff) << 0)
+#       define PRODUCT_ID(x)                             (((x) & 0xffff) << 16)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO1               0x3b
+#       define SINK_DESCRIPTION_LEN(x)                   (((x) & 0xff) << 0)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO2               0x3c
+#       define PORT_ID0(x)                               (((x) & 0xffffffff) << 0)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO3               0x3d
+#       define PORT_ID1(x)                               (((x) & 0xffffffff) << 0)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO4               0x3e
+#       define DESCRIPTION0(x)                           (((x) & 0xff) << 0)
+#       define DESCRIPTION1(x)                           (((x) & 0xff) << 8)
+#       define DESCRIPTION2(x)                           (((x) & 0xff) << 16)
+#       define DESCRIPTION3(x)                           (((x) & 0xff) << 24)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO5               0x3f
+#       define DESCRIPTION4(x)                           (((x) & 0xff) << 0)
+#       define DESCRIPTION5(x)                           (((x) & 0xff) << 8)
+#       define DESCRIPTION6(x)                           (((x) & 0xff) << 16)
+#       define DESCRIPTION7(x)                           (((x) & 0xff) << 24)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO6               0x40
+#       define DESCRIPTION8(x)                           (((x) & 0xff) << 0)
+#       define DESCRIPTION9(x)                           (((x) & 0xff) << 8)
+#       define DESCRIPTION10(x)                          (((x) & 0xff) << 16)
+#       define DESCRIPTION11(x)                          (((x) & 0xff) << 24)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO7               0x41
+#       define DESCRIPTION12(x)                          (((x) & 0xff) << 0)
+#       define DESCRIPTION13(x)                          (((x) & 0xff) << 8)
+#       define DESCRIPTION14(x)                          (((x) & 0xff) << 16)
+#       define DESCRIPTION15(x)                          (((x) & 0xff) << 24)
+#define AZ_F0_CODEC_PIN_CONTROL_SINK_INFO8               0x42
+#       define DESCRIPTION16(x)                          (((x) & 0xff) << 0)
+#       define DESCRIPTION17(x)                          (((x) & 0xff) << 8)
+
 #define AZ_F0_CODEC_PIN_CONTROL_HOTPLUG_CONTROL          0x54
 #       define AUDIO_ENABLED                             (1 << 31)
 
diff --git a/drivers/gpu/drm/shmobile/Kconfig b/drivers/gpu/drm/shmobile/Kconfig
index d1372862d871..2ee44ca9d67f 100644
--- a/drivers/gpu/drm/shmobile/Kconfig
+++ b/drivers/gpu/drm/shmobile/Kconfig
@@ -1,6 +1,7 @@
 config DRM_SHMOBILE
         tristate "DRM Support for SH Mobile"
         depends on DRM && (ARM || SUPERH)
+        select BACKLIGHT_CLASS_DEVICE
         select DRM_KMS_HELPER
         select DRM_KMS_FB_HELPER
         select DRM_KMS_CMA_HELPER
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
index 54bad98e9477..562f9a401cf6 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
@@ -40,7 +40,7 @@
 static void shmob_drm_clk_on(struct shmob_drm_device *sdev)
 {
         if (sdev->clock)
-                clk_enable(sdev->clock);
+                clk_prepare_enable(sdev->clock);
 #if 0
         if (sdev->meram_dev && sdev->meram_dev->pdev)
                 pm_runtime_get_sync(&sdev->meram_dev->pdev->dev);
@@ -54,7 +54,7 @@ static void shmob_drm_clk_off(struct shmob_drm_device *sdev)
                 pm_runtime_put_sync(&sdev->meram_dev->pdev->dev);
 #endif
         if (sdev->clock)
-                clk_disable(sdev->clock);
+                clk_disable_unprepare(sdev->clock);
 }
 
 /* -----------------------------------------------------------------------------
diff --git a/drivers/gpu/host1x/drm/Kconfig b/drivers/gpu/drm/tegra/Kconfig
index 0f36ddd74e87..8961ba6a34b8 100644
--- a/drivers/gpu/host1x/drm/Kconfig
+++ b/drivers/gpu/drm/tegra/Kconfig
@@ -1,6 +1,8 @@
 config DRM_TEGRA
         bool "NVIDIA Tegra DRM"
+        depends on ARCH_TEGRA || ARCH_MULTIPLATFORM
         depends on DRM
+        select TEGRA_HOST1X
         select DRM_KMS_HELPER
         select DRM_KMS_FB_HELPER
         select FB_SYS_FILLRECT
@@ -14,6 +16,11 @@ config DRM_TEGRA
 
 if DRM_TEGRA
 
+config DRM_TEGRA_DEBUG
+        bool "NVIDIA Tegra DRM debug support"
+        help
+          Say yes here to enable debugging support.
+
 config DRM_TEGRA_STAGING
         bool "Enable HOST1X interface"
         depends on STAGING
@@ -22,9 +29,4 @@ config DRM_TEGRA_STAGING
 
           If unsure, choose N.
 
-config DRM_TEGRA_DEBUG
-        bool "NVIDIA Tegra DRM debug support"
-        help
-          Say yes here to enable debugging support.
-
 endif
diff --git a/drivers/gpu/drm/tegra/Makefile b/drivers/gpu/drm/tegra/Makefile
new file mode 100644
index 000000000000..edc76abd58bb
--- /dev/null
+++ b/drivers/gpu/drm/tegra/Makefile
@@ -0,0 +1,15 @@
+ccflags-$(CONFIG_DRM_TEGRA_DEBUG) += -DDEBUG
+
+tegra-drm-y := \
+        bus.o \
+        drm.o \
+        gem.o \
+        fb.o \
+        dc.o \
+        output.o \
+        rgb.o \
+        hdmi.o \
+        gr2d.o \
+        gr3d.o
+
+obj-$(CONFIG_DRM_TEGRA) += tegra-drm.o
diff --git a/drivers/gpu/drm/tegra/bus.c b/drivers/gpu/drm/tegra/bus.c
new file mode 100644
index 000000000000..565f8f7b9a47
--- /dev/null
+++ b/drivers/gpu/drm/tegra/bus.c
@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2013 NVIDIA Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include "drm.h"
+
+static int drm_host1x_set_busid(struct drm_device *dev,
+                                struct drm_master *master)
+{
+        const char *device = dev_name(dev->dev);
+        const char *driver = dev->driver->name;
+        const char *bus = dev->dev->bus->name;
+        int length;
+
+        master->unique_len = strlen(bus) + 1 + strlen(device);
+        master->unique_size = master->unique_len;
+
+        master->unique = kmalloc(master->unique_len + 1, GFP_KERNEL);
+        if (!master->unique)
+                return -ENOMEM;
+
+        snprintf(master->unique, master->unique_len + 1, "%s:%s", bus, device);
+
+        length = strlen(driver) + 1 + master->unique_len;
+
+        dev->devname = kmalloc(length + 1, GFP_KERNEL);
+        if (!dev->devname)
+                return -ENOMEM;
+
+        snprintf(dev->devname, length + 1, "%s@%s", driver, master->unique);
+
+        return 0;
+}
+
+static struct drm_bus drm_host1x_bus = {
+        .bus_type = DRIVER_BUS_HOST1X,
+        .set_busid = drm_host1x_set_busid,
+};
+
+int drm_host1x_init(struct drm_driver *driver, struct host1x_device *device)
+{
+        struct drm_device *drm;
+        int ret;
+
+        INIT_LIST_HEAD(&driver->device_list);
+        driver->bus = &drm_host1x_bus;
+
+        drm = drm_dev_alloc(driver, &device->dev);
+        if (!drm)
+                return -ENOMEM;
+
+        ret = drm_dev_register(drm, 0);
+        if (ret)
+                goto err_free;
+
+        DRM_INFO("Initialized %s %d.%d.%d %s on minor %d\n", driver->name,
+                 driver->major, driver->minor, driver->patchlevel,
+                 driver->date, drm->primary->index);
+
+        return 0;
+
+err_free:
+        drm_dev_free(drm);
+        return ret;
+}
+
+void drm_host1x_exit(struct drm_driver *driver, struct host1x_device *device)
+{
+        struct tegra_drm *tegra = dev_get_drvdata(&device->dev);
+
+        drm_put_dev(tegra->drm);
+}
diff --git a/drivers/gpu/host1x/drm/dc.c b/drivers/gpu/drm/tegra/dc.c
index b1a05ad901c3..ae1cb31ead7e 100644
--- a/drivers/gpu/host1x/drm/dc.c
+++ b/drivers/gpu/drm/tegra/dc.c
@@ -8,13 +8,9 @@
  */
 
 #include <linux/clk.h>
-#include <linux/debugfs.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
 #include <linux/clk/tegra.h>
+#include <linux/debugfs.h>
 
-#include "host1x_client.h"
 #include "dc.h"
 #include "drm.h"
 #include "gem.h"
@@ -51,6 +47,8 @@ static int tegra_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
         window.dst.h = crtc_h;
         window.format = tegra_dc_format(fb->pixel_format);
         window.bits_per_pixel = fb->bits_per_pixel;
+        window.bottom_up = tegra_fb_is_bottom_up(fb);
+        window.tiled = tegra_fb_is_tiled(fb);
 
         for (i = 0; i < drm_format_num_planes(fb->pixel_format); i++) {
                 struct tegra_bo *bo = tegra_fb_get_plane(fb, i);
@@ -97,8 +95,11 @@ static int tegra_plane_disable(struct drm_plane *plane)
 
 static void tegra_plane_destroy(struct drm_plane *plane)
 {
+        struct tegra_plane *p = to_tegra_plane(plane);
+
         tegra_plane_disable(plane);
         drm_plane_cleanup(plane);
+        kfree(p);
 }
 
 static const struct drm_plane_funcs tegra_plane_funcs = {
@@ -124,7 +125,7 @@ static int tegra_dc_add_planes(struct drm_device *drm, struct tegra_dc *dc)
         for (i = 0; i < 2; i++) {
                 struct tegra_plane *plane;
 
-                plane = devm_kzalloc(drm->dev, sizeof(*plane), GFP_KERNEL);
+                plane = kzalloc(sizeof(*plane), GFP_KERNEL);
                 if (!plane)
                         return -ENOMEM;
 
@@ -133,8 +134,10 @@ static int tegra_dc_add_planes(struct drm_device *drm, struct tegra_dc *dc)
                 err = drm_plane_init(drm, &plane->base, 1 << dc->pipe,
                                      &tegra_plane_funcs, plane_formats,
                                      ARRAY_SIZE(plane_formats), false);
-                if (err < 0)
+                if (err < 0) {
+                        kfree(plane);
                         return err;
+                }
         }
 
         return 0;
@@ -145,6 +148,7 @@ static int tegra_dc_set_base(struct tegra_dc *dc, int x, int y,
 {
         unsigned int format = tegra_dc_format(fb->pixel_format);
         struct tegra_bo *bo = tegra_fb_get_plane(fb, 0);
+        unsigned int h_offset = 0, v_offset = 0;
         unsigned long value;
 
         tegra_dc_writel(dc, WINDOW_A_SELECT, DC_CMD_DISPLAY_WINDOW_HEADER);
@@ -156,6 +160,32 @@ static int tegra_dc_set_base(struct tegra_dc *dc, int x, int y,
         tegra_dc_writel(dc, fb->pitches[0], DC_WIN_LINE_STRIDE);
         tegra_dc_writel(dc, format, DC_WIN_COLOR_DEPTH);
 
+        if (tegra_fb_is_tiled(fb)) {
+                value = DC_WIN_BUFFER_ADDR_MODE_TILE_UV |
+                        DC_WIN_BUFFER_ADDR_MODE_TILE;
+        } else {
+                value = DC_WIN_BUFFER_ADDR_MODE_LINEAR_UV |
+                        DC_WIN_BUFFER_ADDR_MODE_LINEAR;
+        }
+
+        tegra_dc_writel(dc, value, DC_WIN_BUFFER_ADDR_MODE);
+
+        /* make sure bottom-up buffers are properly displayed */
+        if (tegra_fb_is_bottom_up(fb)) {
+                value = tegra_dc_readl(dc, DC_WIN_WIN_OPTIONS);
+                value |= INVERT_V;
+                tegra_dc_writel(dc, value, DC_WIN_WIN_OPTIONS);
+
+                v_offset += fb->height - 1;
+        } else {
+                value = tegra_dc_readl(dc, DC_WIN_WIN_OPTIONS);
+                value &= ~INVERT_V;
+                tegra_dc_writel(dc, value, DC_WIN_WIN_OPTIONS);
+        }
+
+        tegra_dc_writel(dc, h_offset, DC_WINBUF_ADDR_H_OFFSET);
+        tegra_dc_writel(dc, v_offset, DC_WINBUF_ADDR_V_OFFSET);
+
         value = GENERAL_UPDATE | WIN_A_UPDATE;
         tegra_dc_writel(dc, value, DC_CMD_STATE_CONTROL);
 
@@ -255,14 +285,26 @@ static int tegra_dc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         return 0;
 }
 
+static void drm_crtc_clear(struct drm_crtc *crtc)
+{
+        memset(crtc, 0, sizeof(*crtc));
+}
+
+static void tegra_dc_destroy(struct drm_crtc *crtc)
+{
+        drm_crtc_cleanup(crtc);
+        drm_crtc_clear(crtc);
+}
+
 static const struct drm_crtc_funcs tegra_crtc_funcs = {
         .page_flip = tegra_dc_page_flip,
         .set_config = drm_crtc_helper_set_config,
-        .destroy = drm_crtc_cleanup,
+        .destroy = tegra_dc_destroy,
 };
 
 static void tegra_crtc_disable(struct drm_crtc *crtc)
 {
+        struct tegra_dc *dc = to_tegra_dc(crtc);
         struct drm_device *drm = crtc->dev;
         struct drm_plane *plane;
 
@@ -277,6 +319,8 @@ static void tegra_crtc_disable(struct drm_crtc *crtc)
                         }
                 }
         }
+
+        drm_vblank_off(drm, dc->pipe);
 }
 
 static bool tegra_crtc_mode_fixup(struct drm_crtc *crtc,
@@ -491,9 +535,22 @@ int tegra_dc_setup_window(struct tegra_dc *dc, unsigned int index,
                 tegra_dc_writel(dc, window->stride[0], DC_WIN_LINE_STRIDE);
         }
 
+        if (window->bottom_up)
+                v_offset += window->src.h - 1;
+
         tegra_dc_writel(dc, h_offset, DC_WINBUF_ADDR_H_OFFSET);
         tegra_dc_writel(dc, v_offset, DC_WINBUF_ADDR_V_OFFSET);
 
+        if (window->tiled) {
+                value = DC_WIN_BUFFER_ADDR_MODE_TILE_UV |
+                        DC_WIN_BUFFER_ADDR_MODE_TILE;
+        } else {
+                value = DC_WIN_BUFFER_ADDR_MODE_LINEAR_UV |
+                        DC_WIN_BUFFER_ADDR_MODE_LINEAR;
+        }
+
+        tegra_dc_writel(dc, value, DC_WIN_BUFFER_ADDR_MODE);
+
         value = WIN_ENABLE;
 
         if (yuv) {
@@ -512,6 +569,9 @@ int tegra_dc_setup_window(struct tegra_dc *dc, unsigned int index,
                 value |= COLOR_EXPAND;
         }
 
+        if (window->bottom_up)
+                value |= INVERT_V;
+
         tegra_dc_writel(dc, value, DC_WIN_WIN_OPTIONS);
 
         /*
@@ -1041,30 +1101,30 @@ static int tegra_dc_debugfs_exit(struct tegra_dc *dc)
         return 0;
 }
 
-static int tegra_dc_drm_init(struct host1x_client *client,
-                             struct drm_device *drm)
+static int tegra_dc_init(struct host1x_client *client)
 {
+        struct tegra_drm *tegra = dev_get_drvdata(client->parent);
         struct tegra_dc *dc = host1x_client_to_dc(client);
         int err;
 
-        dc->pipe = drm->mode_config.num_crtc;
+        dc->pipe = tegra->drm->mode_config.num_crtc;
 
-        drm_crtc_init(drm, &dc->base, &tegra_crtc_funcs);
+        drm_crtc_init(tegra->drm, &dc->base, &tegra_crtc_funcs);
         drm_mode_crtc_set_gamma_size(&dc->base, 256);
         drm_crtc_helper_add(&dc->base, &tegra_crtc_helper_funcs);
 
-        err = tegra_dc_rgb_init(drm, dc);
+        err = tegra_dc_rgb_init(tegra->drm, dc);
         if (err < 0 && err != -ENODEV) {
                 dev_err(dc->dev, "failed to initialize RGB output: %d\n", err);
                 return err;
         }
 
-        err = tegra_dc_add_planes(drm, dc);
+        err = tegra_dc_add_planes(tegra->drm, dc);
         if (err < 0)
                 return err;
 
         if (IS_ENABLED(CONFIG_DEBUG_FS)) {
-                err = tegra_dc_debugfs_init(dc, drm->primary);
+                err = tegra_dc_debugfs_init(dc, tegra->drm->primary);
                 if (err < 0)
                         dev_err(dc->dev, "debugfs setup failed: %d\n", err);
         }
@@ -1080,7 +1140,7 @@ static int tegra_dc_drm_init(struct host1x_client *client,
         return 0;
 }
 
-static int tegra_dc_drm_exit(struct host1x_client *client)
+static int tegra_dc_exit(struct host1x_client *client)
 {
         struct tegra_dc *dc = host1x_client_to_dc(client);
         int err;
@@ -1103,13 +1163,12 @@ static int tegra_dc_drm_exit(struct host1x_client *client)
 }
 
 static const struct host1x_client_ops dc_client_ops = {
-        .drm_init = tegra_dc_drm_init,
-        .drm_exit = tegra_dc_drm_exit,
+        .init = tegra_dc_init,
+        .exit = tegra_dc_exit,
 };
 
 static int tegra_dc_probe(struct platform_device *pdev)
 {
-        struct host1x_drm *host1x = host1x_get_drm_data(pdev->dev.parent);
         struct resource *regs;
         struct tegra_dc *dc;
         int err;
@@ -1153,7 +1212,7 @@ static int tegra_dc_probe(struct platform_device *pdev)
                 return err;
         }
 
-        err = host1x_register_client(host1x, &dc->client);
+        err = host1x_client_register(&dc->client);
         if (err < 0) {
                 dev_err(&pdev->dev, "failed to register host1x client: %d\n",
                         err);
@@ -1167,17 +1226,22 @@ static int tegra_dc_probe(struct platform_device *pdev)
 
 static int tegra_dc_remove(struct platform_device *pdev)
 {
-        struct host1x_drm *host1x = host1x_get_drm_data(pdev->dev.parent);
         struct tegra_dc *dc = platform_get_drvdata(pdev);
         int err;
 
-        err = host1x_unregister_client(host1x, &dc->client);
+        err = host1x_client_unregister(&dc->client);
         if (err < 0) {
                 dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
                         err);
                 return err;
         }
 
+        err = tegra_dc_rgb_remove(dc);
+        if (err < 0) {
+                dev_err(&pdev->dev, "failed to remove RGB output: %d\n", err);
+                return err;
+        }
+
         clk_disable_unprepare(dc->clk);
 
         return 0;
diff --git a/drivers/gpu/host1x/drm/dc.h b/drivers/gpu/drm/tegra/dc.h
index 79eaec9aac77..91bbda291470 100644
--- a/drivers/gpu/host1x/drm/dc.h
+++ b/drivers/gpu/drm/tegra/dc.h
@@ -302,6 +302,7 @@
 #define DC_WIN_CSC_KVB                          0x618
 
 #define DC_WIN_WIN_OPTIONS                      0x700
+#define INVERT_V     (1 <<  2)
 #define COLOR_EXPAND (1 <<  6)
 #define CSC_ENABLE   (1 << 18)
 #define WIN_ENABLE   (1 << 30)
@@ -365,6 +366,10 @@
 #define DC_WIN_BUF_STRIDE                       0x70b
 #define DC_WIN_UV_BUF_STRIDE                    0x70c
 #define DC_WIN_BUFFER_ADDR_MODE                 0x70d
+#define DC_WIN_BUFFER_ADDR_MODE_LINEAR          (0 <<  0)
+#define DC_WIN_BUFFER_ADDR_MODE_TILE            (1 <<  0)
+#define DC_WIN_BUFFER_ADDR_MODE_LINEAR_UV       (0 << 16)
+#define DC_WIN_BUFFER_ADDR_MODE_TILE_UV         (1 << 16)
 #define DC_WIN_DV_CONTROL                       0x70e
 
 #define DC_WIN_BLEND_NOKEY                      0x70f
diff --git a/drivers/gpu/host1x/drm/drm.c b/drivers/gpu/drm/tegra/drm.c
index df7d90a3a4fa..28e178137718 100644
--- a/drivers/gpu/host1x/drm/drm.c
+++ b/drivers/gpu/drm/tegra/drm.c
@@ -7,21 +7,10 @@
  * published by the Free Software Foundation.
  */
 
-#include <linux/module.h>
-#include <linux/of_address.h>
-#include <linux/of_platform.h>
+#include <linux/host1x.h>
 
-#include <linux/dma-mapping.h>
-#include <asm/dma-iommu.h>
-
-#include <drm/drm.h>
-#include <drm/drmP.h>
-
-#include "host1x_client.h"
-#include "dev.h"
 #include "drm.h"
 #include "gem.h"
-#include "syncpt.h"
 
 #define DRIVER_NAME "tegra"
 #define DRIVER_DESC "NVIDIA Tegra graphics"
@@ -30,297 +19,235 @@
 #define DRIVER_MINOR 0
 #define DRIVER_PATCHLEVEL 0
 
-struct host1x_drm_client {
-        struct host1x_client *client;
-        struct device_node *np;
-        struct list_head list;
+struct tegra_drm_file {
+        struct list_head contexts;
 };
 
-static int host1x_add_drm_client(struct host1x_drm *host1x,
-                                 struct device_node *np)
+static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
 {
-        struct host1x_drm_client *client;
+        struct host1x_device *device = to_host1x_device(drm->dev);
+        struct tegra_drm *tegra;
+        int err;
 
-        client = kzalloc(sizeof(*client), GFP_KERNEL);
-        if (!client)
+        tegra = kzalloc(sizeof(*tegra), GFP_KERNEL);
+        if (!tegra)
                 return -ENOMEM;
 
-        INIT_LIST_HEAD(&client->list);
-        client->np = of_node_get(np);
+        dev_set_drvdata(drm->dev, tegra);
+        mutex_init(&tegra->clients_lock);
+        INIT_LIST_HEAD(&tegra->clients);
+        drm->dev_private = tegra;
+        tegra->drm = drm;
 
-        list_add_tail(&client->list, &host1x->drm_clients);
+        drm_mode_config_init(drm);
 
-        return 0;
-}
+        err = host1x_device_init(device);
+        if (err < 0)
+                return err;
 
-static int host1x_activate_drm_client(struct host1x_drm *host1x,
-                                      struct host1x_drm_client *drm,
-                                      struct host1x_client *client)
-{
-        mutex_lock(&host1x->drm_clients_lock);
-        list_del_init(&drm->list);
-        list_add_tail(&drm->list, &host1x->drm_active);
-        drm->client = client;
-        mutex_unlock(&host1x->drm_clients_lock);
+        /*
+         * We don't use the drm_irq_install() helpers provided by the DRM
+         * core, so we need to set this manually in order to allow the
+         * DRM_IOCTL_WAIT_VBLANK to operate correctly.
+         */
+        drm->irq_enabled = true;
 
-        return 0;
-}
+        err = drm_vblank_init(drm, drm->mode_config.num_crtc);
+        if (err < 0)
+                return err;
 
-static int host1x_remove_drm_client(struct host1x_drm *host1x,
-                                    struct host1x_drm_client *client)
-{
-        mutex_lock(&host1x->drm_clients_lock);
-        list_del_init(&client->list);
-        mutex_unlock(&host1x->drm_clients_lock);
+        err = tegra_drm_fb_init(drm);
+        if (err < 0)
+                return err;
 
-        of_node_put(client->np);
-        kfree(client);
+        drm_kms_helper_poll_init(drm);
 
         return 0;
 }
 
-static int host1x_parse_dt(struct host1x_drm *host1x)
+static int tegra_drm_unload(struct drm_device *drm)
 {
-        static const char * const compat[] = {
-                "nvidia,tegra20-dc",
-                "nvidia,tegra20-hdmi",
-                "nvidia,tegra20-gr2d",
-                "nvidia,tegra30-dc",
-                "nvidia,tegra30-hdmi",
-                "nvidia,tegra30-gr2d",
-        };
-        unsigned int i;
+        struct host1x_device *device = to_host1x_device(drm->dev);
         int err;
 
-        for (i = 0; i < ARRAY_SIZE(compat); i++) {
-                struct device_node *np;
+        drm_kms_helper_poll_fini(drm);
+        tegra_drm_fb_exit(drm);
+        drm_vblank_cleanup(drm);
+        drm_mode_config_cleanup(drm);
 
-                for_each_child_of_node(host1x->dev->of_node, np) {
-                        if (of_device_is_compatible(np, compat[i]) &&
-                            of_device_is_available(np)) {
-                                err = host1x_add_drm_client(host1x, np);
-                                if (err < 0)
-                                        return err;
-                        }
-                }
-        }
+        err = host1x_device_exit(device);
+        if (err < 0)
+                return err;
 
         return 0;
 }
 
-int host1x_drm_alloc(struct platform_device *pdev)
+static int tegra_drm_open(struct drm_device *drm, struct drm_file *filp)
 {
-        struct host1x_drm *host1x;
-        int err;
+        struct tegra_drm_file *fpriv;
 
-        host1x = devm_kzalloc(&pdev->dev, sizeof(*host1x), GFP_KERNEL);
-        if (!host1x)
+        fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL);
+        if (!fpriv)
                 return -ENOMEM;
 
-        mutex_init(&host1x->drm_clients_lock);
-        INIT_LIST_HEAD(&host1x->drm_clients);
-        INIT_LIST_HEAD(&host1x->drm_active);
-        mutex_init(&host1x->clients_lock);
-        INIT_LIST_HEAD(&host1x->clients);
-        host1x->dev = &pdev->dev;
-
-        err = host1x_parse_dt(host1x);
-        if (err < 0) {
-                dev_err(&pdev->dev, "failed to parse DT: %d\n", err);
-                return err;
-        }
-
-        host1x_set_drm_data(&pdev->dev, host1x);
+        INIT_LIST_HEAD(&fpriv->contexts);
+        filp->driver_priv = fpriv;
 
         return 0;
 }
 
-int host1x_drm_init(struct host1x_drm *host1x, struct drm_device *drm)
+static void tegra_drm_context_free(struct tegra_drm_context *context)
 {
-        struct host1x_client *client;
-
-        mutex_lock(&host1x->clients_lock);
-
-        list_for_each_entry(client, &host1x->clients, list) {
-                if (client->ops && client->ops->drm_init) {
-                        int err = client->ops->drm_init(client, drm);
-                        if (err < 0) {
-                                dev_err(host1x->dev,
-                                        "DRM setup failed for %s: %d\n",
-                                        dev_name(client->dev), err);
-                                mutex_unlock(&host1x->clients_lock);
-                                return err;
-                        }
-                }
-        }
-
-        mutex_unlock(&host1x->clients_lock);
-
-        return 0;
+        context->client->ops->close_channel(context);
+        kfree(context);
 }
 
-int host1x_drm_exit(struct host1x_drm *host1x)
+static void tegra_drm_lastclose(struct drm_device *drm)
 {
-        struct platform_device *pdev = to_platform_device(host1x->dev);
-        struct host1x_client *client;
-
-        if (!host1x->drm)
-                return 0;
-
-        mutex_lock(&host1x->clients_lock);
+        struct tegra_drm *tegra = drm->dev_private;
 
-        list_for_each_entry_reverse(client, &host1x->clients, list) {
-                if (client->ops && client->ops->drm_exit) {
-                        int err = client->ops->drm_exit(client);
-                        if (err < 0) {
-                                dev_err(host1x->dev,
-                                        "DRM cleanup failed for %s: %d\n",
-                                        dev_name(client->dev), err);
-                                mutex_unlock(&host1x->clients_lock);
-                                return err;
-                        }
-                }
-        }
+        tegra_fbdev_restore_mode(tegra->fbdev);
+}
 
-        mutex_unlock(&host1x->clients_lock);
+static struct host1x_bo *
+host1x_bo_lookup(struct drm_device *drm, struct drm_file *file, u32 handle)
+{
+        struct drm_gem_object *gem;
+        struct tegra_bo *bo;
 
-        drm_platform_exit(&tegra_drm_driver, pdev);
-        host1x->drm = NULL;
+        gem = drm_gem_object_lookup(drm, file, handle);
+        if (!gem)
+                return NULL;
 
-        return 0;
-}
+        mutex_lock(&drm->struct_mutex);
+        drm_gem_object_unreference(gem);
+        mutex_unlock(&drm->struct_mutex);
 
-int host1x_register_client(struct host1x_drm *host1x,
-                           struct host1x_client *client)
-{
-        struct host1x_drm_client *drm, *tmp;
+        bo = to_tegra_bo(gem);
+        return &bo->base;
+}
+
+int tegra_drm_submit(struct tegra_drm_context *context,
+                     struct drm_tegra_submit *args, struct drm_device *drm,
+                     struct drm_file *file)
+{
+        unsigned int num_cmdbufs = args->num_cmdbufs;
+        unsigned int num_relocs = args->num_relocs;
+        unsigned int num_waitchks = args->num_waitchks;
+        struct drm_tegra_cmdbuf __user *cmdbufs =
+                (void * __user)(uintptr_t)args->cmdbufs;
+        struct drm_tegra_reloc __user *relocs =
+                (void * __user)(uintptr_t)args->relocs;
+        struct drm_tegra_waitchk __user *waitchks =
+                (void * __user)(uintptr_t)args->waitchks;
+        struct drm_tegra_syncpt syncpt;
+        struct host1x_job *job;
         int err;
 
-        mutex_lock(&host1x->clients_lock);
-        list_add_tail(&client->list, &host1x->clients);
-        mutex_unlock(&host1x->clients_lock);
+        /* We don't yet support other than one syncpt_incr struct per submit */
+        if (args->num_syncpts != 1)
+                return -EINVAL;
 
-        list_for_each_entry_safe(drm, tmp, &host1x->drm_clients, list)
-                if (drm->np == client->dev->of_node)
-                        host1x_activate_drm_client(host1x, drm, client);
+        job = host1x_job_alloc(context->channel, args->num_cmdbufs,
+                               args->num_relocs, args->num_waitchks);
+        if (!job)
+                return -ENOMEM;
 
-        if (list_empty(&host1x->drm_clients)) {
-                struct platform_device *pdev = to_platform_device(host1x->dev);
+        job->num_relocs = args->num_relocs;
+        job->num_waitchk = args->num_waitchks;
+        job->client = (u32)args->context;
+        job->class = context->client->base.class;
+        job->serialize = true;
 
-                err = drm_platform_init(&tegra_drm_driver, pdev);
-                if (err < 0) {
-                        dev_err(host1x->dev, "drm_platform_init(): %d\n", err);
-                        return err;
-                }
-        }
+        while (num_cmdbufs) {
+                struct drm_tegra_cmdbuf cmdbuf;
+                struct host1x_bo *bo;
 
-        return 0;
-}
+                err = copy_from_user(&cmdbuf, cmdbufs, sizeof(cmdbuf));
+                if (err)
+                        goto fail;
 
-int host1x_unregister_client(struct host1x_drm *host1x,
-                             struct host1x_client *client)
-{
-        struct host1x_drm_client *drm, *tmp;
-        int err;
-
-        list_for_each_entry_safe(drm, tmp, &host1x->drm_active, list) {
-                if (drm->client == client) {
-                        err = host1x_drm_exit(host1x);
-                        if (err < 0) {
-                                dev_err(host1x->dev, "host1x_drm_exit(): %d\n",
-                                        err);
-                                return err;
-                        }
-
-                        host1x_remove_drm_client(host1x, drm);
-                        break;
+                bo = host1x_bo_lookup(drm, file, cmdbuf.handle);
+                if (!bo) {
+                        err = -ENOENT;
+                        goto fail;
                 }
+
+                host1x_job_add_gather(job, bo, cmdbuf.words, cmdbuf.offset);
+                num_cmdbufs--;
+                cmdbufs++;
         }
 
-        mutex_lock(&host1x->clients_lock);
-        list_del_init(&client->list);
-        mutex_unlock(&host1x->clients_lock);
+        err = copy_from_user(job->relocarray, relocs,
+                             sizeof(*relocs) * num_relocs);
+        if (err)
+                goto fail;
 
-        return 0;
-}
+        while (num_relocs--) {
+                struct host1x_reloc *reloc = &job->relocarray[num_relocs];
+                struct host1x_bo *cmdbuf, *target;
 
-static int tegra_drm_load(struct drm_device *drm, unsigned long flags)
-{
-        struct host1x_drm *host1x;
-        int err;
+                cmdbuf = host1x_bo_lookup(drm, file, (u32)reloc->cmdbuf);
+                target = host1x_bo_lookup(drm, file, (u32)reloc->target);
 
-        host1x = host1x_get_drm_data(drm->dev);
-        drm->dev_private = host1x;
-        host1x->drm = drm;
+                reloc->cmdbuf = cmdbuf;
+                reloc->target = target;
 
-        drm_mode_config_init(drm);
+                if (!reloc->target || !reloc->cmdbuf) {
+                        err = -ENOENT;
+                        goto fail;
+                }
+        }
 
-        err = host1x_drm_init(host1x, drm);
-        if (err < 0)
-                return err;
+        err = copy_from_user(job->waitchk, waitchks,
+                             sizeof(*waitchks) * num_waitchks);
+        if (err)
+                goto fail;
 
-        /*
-         * We don't use the drm_irq_install() helpers provided by the DRM
-         * core, so we need to set this manually in order to allow the
-         * DRM_IOCTL_WAIT_VBLANK to operate correctly.
-         */
-        drm->irq_enabled = true;
+        err = copy_from_user(&syncpt, (void * __user)(uintptr_t)args->syncpts,
+                             sizeof(syncpt));
+        if (err)
+                goto fail;
 
-        err = drm_vblank_init(drm, drm->mode_config.num_crtc);
-        if (err < 0)
-                return err;
+        job->is_addr_reg = context->client->ops->is_addr_reg;
+        job->syncpt_incrs = syncpt.incrs;
+        job->syncpt_id = syncpt.id;
+        job->timeout = 10000;
 
-        err = tegra_drm_fb_init(drm);
-        if (err < 0)
-                return err;
+        if (args->timeout && args->timeout < 10000)
+                job->timeout = args->timeout;
 
-        drm_kms_helper_poll_init(drm);
+        err = host1x_job_pin(job, context->client->base.dev);
+        if (err)
+                goto fail;
 
-        return 0;
-}
+        err = host1x_job_submit(job);
+        if (err)
+                goto fail_submit;
 
-static int tegra_drm_unload(struct drm_device *drm)
-{
-        drm_kms_helper_poll_fini(drm);
-        tegra_drm_fb_exit(drm);
-
-        drm_mode_config_cleanup(drm);
+        args->fence = job->syncpt_end;
 
+        host1x_job_put(job);
         return 0;
-}
 
-static int tegra_drm_open(struct drm_device *drm, struct drm_file *filp)
-{
-        struct host1x_drm_file *fpriv;
-
-        fpriv = kzalloc(sizeof(*fpriv), GFP_KERNEL);
-        if (!fpriv)
-                return -ENOMEM;
-
-        INIT_LIST_HEAD(&fpriv->contexts);
-        filp->driver_priv = fpriv;
-
-        return 0;
+fail_submit:
+        host1x_job_unpin(job);
+fail:
+        host1x_job_put(job);
+        return err;
 }
 
-static void host1x_drm_context_free(struct host1x_drm_context *context)
-{
-        context->client->ops->close_channel(context);
-        kfree(context);
-}
 
-static void tegra_drm_lastclose(struct drm_device *drm)
+#ifdef CONFIG_DRM_TEGRA_STAGING
+static struct tegra_drm_context *tegra_drm_get_context(__u64 context)
 {
-        struct host1x_drm *host1x = drm->dev_private;
-
-        tegra_fbdev_restore_mode(host1x->fbdev);
+        return (struct tegra_drm_context *)(uintptr_t)context;
 }
 
-#ifdef CONFIG_DRM_TEGRA_STAGING
-static bool host1x_drm_file_owns_context(struct host1x_drm_file *file,
-                                         struct host1x_drm_context *context)
+static bool tegra_drm_file_owns_context(struct tegra_drm_file *file,
+                                        struct tegra_drm_context *context)
 {
-        struct host1x_drm_context *ctx;
+        struct tegra_drm_context *ctx;
 
         list_for_each_entry(ctx, &file->contexts, list)
                 if (ctx == context)
@@ -335,7 +262,7 @@ static int tegra_gem_create(struct drm_device *drm, void *data,
         struct drm_tegra_gem_create *args = data;
         struct tegra_bo *bo;
 
-        bo = tegra_bo_create_with_handle(file, drm, args->size,
+        bo = tegra_bo_create_with_handle(file, drm, args->size, args->flags,
                                          &args->handle);
         if (IS_ERR(bo))
                 return PTR_ERR(bo);
@@ -366,10 +293,11 @@ static int tegra_gem_mmap(struct drm_device *drm, void *data,
 static int tegra_syncpt_read(struct drm_device *drm, void *data,
                              struct drm_file *file)
 {
+        struct host1x *host = dev_get_drvdata(drm->dev->parent);
         struct drm_tegra_syncpt_read *args = data;
-        struct host1x *host = dev_get_drvdata(drm->dev);
-        struct host1x_syncpt *sp = host1x_syncpt_get(host, args->id);
+        struct host1x_syncpt *sp;
 
+        sp = host1x_syncpt_get(host, args->id);
         if (!sp)
                 return -EINVAL;
 
@@ -380,10 +308,11 @@ static int tegra_syncpt_read(struct drm_device *drm, void *data,
 static int tegra_syncpt_incr(struct drm_device *drm, void *data,
                              struct drm_file *file)
 {
+        struct host1x *host1x = dev_get_drvdata(drm->dev->parent);
         struct drm_tegra_syncpt_incr *args = data;
-        struct host1x *host = dev_get_drvdata(drm->dev);
-        struct host1x_syncpt *sp = host1x_syncpt_get(host, args->id);
+        struct host1x_syncpt *sp;
 
+        sp = host1x_syncpt_get(host1x, args->id);
         if (!sp)
                 return -EINVAL;
 
@@ -393,10 +322,11 @@ static int tegra_syncpt_incr(struct drm_device *drm, void *data,
 static int tegra_syncpt_wait(struct drm_device *drm, void *data,
                              struct drm_file *file)
 {
+        struct host1x *host1x = dev_get_drvdata(drm->dev->parent);
         struct drm_tegra_syncpt_wait *args = data;
-        struct host1x *host = dev_get_drvdata(drm->dev);
-        struct host1x_syncpt *sp = host1x_syncpt_get(host, args->id);
+        struct host1x_syncpt *sp;
 
+        sp = host1x_syncpt_get(host1x, args->id);
         if (!sp)
                 return -EINVAL;
 
@@ -407,26 +337,26 @@ static int tegra_syncpt_wait(struct drm_device *drm, void *data,
 static int tegra_open_channel(struct drm_device *drm, void *data,
                               struct drm_file *file)
 {
+        struct tegra_drm_file *fpriv = file->driver_priv;
+        struct tegra_drm *tegra = drm->dev_private;
         struct drm_tegra_open_channel *args = data;
-        struct host1x_client *client;
-        struct host1x_drm_context *context;
-        struct host1x_drm_file *fpriv = file->driver_priv;
-        struct host1x_drm *host1x = drm->dev_private;
+        struct tegra_drm_context *context;
+        struct tegra_drm_client *client;
         int err = -ENODEV;
 
         context = kzalloc(sizeof(*context), GFP_KERNEL);
         if (!context)
                 return -ENOMEM;
 
-        list_for_each_entry(client, &host1x->clients, list)
-                if (client->class == args->client) {
+        list_for_each_entry(client, &tegra->clients, list)
+                if (client->base.class == args->client) {
                         err = client->ops->open_channel(client, context);
                         if (err)
                                 break;
 
-                        context->client = client;
                         list_add(&context->list, &fpriv->contexts);
                         args->context = (uintptr_t)context;
+                        context->client = client;
                         return 0;
                 }
 
@@ -437,16 +367,17 @@ static int tegra_open_channel(struct drm_device *drm, void *data,
 static int tegra_close_channel(struct drm_device *drm, void *data,
                                struct drm_file *file)
 {
+        struct tegra_drm_file *fpriv = file->driver_priv;
         struct drm_tegra_close_channel *args = data;
-        struct host1x_drm_file *fpriv = file->driver_priv;
-        struct host1x_drm_context *context =
-                (struct host1x_drm_context *)(uintptr_t)args->context;
+        struct tegra_drm_context *context;
+
+        context = tegra_drm_get_context(args->context);
 
-        if (!host1x_drm_file_owns_context(fpriv, context))
+        if (!tegra_drm_file_owns_context(fpriv, context))
                 return -EINVAL;
 
         list_del(&context->list);
-        host1x_drm_context_free(context);
+        tegra_drm_context_free(context);
 
         return 0;
 }
@@ -454,19 +385,20 @@ static int tegra_close_channel(struct drm_device *drm, void *data,
 static int tegra_get_syncpt(struct drm_device *drm, void *data,
                             struct drm_file *file)
 {
+        struct tegra_drm_file *fpriv = file->driver_priv;
         struct drm_tegra_get_syncpt *args = data;
-        struct host1x_drm_file *fpriv = file->driver_priv;
-        struct host1x_drm_context *context =
-                (struct host1x_drm_context *)(uintptr_t)args->context;
+        struct tegra_drm_context *context;
         struct host1x_syncpt *syncpt;
 
-        if (!host1x_drm_file_owns_context(fpriv, context))
+        context = tegra_drm_get_context(args->context);
+
+        if (!tegra_drm_file_owns_context(fpriv, context))
                 return -ENODEV;
 
-        if (args->index >= context->client->num_syncpts)
+        if (args->index >= context->client->base.num_syncpts)
                 return -EINVAL;
 
-        syncpt = context->client->syncpts[args->index];
+        syncpt = context->client->base.syncpts[args->index];
         args->id = host1x_syncpt_id(syncpt);
 
         return 0;
@@ -475,16 +407,45 @@ static int tegra_get_syncpt(struct drm_device *drm, void *data,
 static int tegra_submit(struct drm_device *drm, void *data,
                         struct drm_file *file)
 {
+        struct tegra_drm_file *fpriv = file->driver_priv;
         struct drm_tegra_submit *args = data;
-        struct host1x_drm_file *fpriv = file->driver_priv;
-        struct host1x_drm_context *context =
-                (struct host1x_drm_context *)(uintptr_t)args->context;
+        struct tegra_drm_context *context;
 
-        if (!host1x_drm_file_owns_context(fpriv, context))
+        context = tegra_drm_get_context(args->context);
+
+        if (!tegra_drm_file_owns_context(fpriv, context))
                 return -ENODEV;
 
         return context->client->ops->submit(context, args, drm, file);
 }
+
+static int tegra_get_syncpt_base(struct drm_device *drm, void *data,
+                                 struct drm_file *file)
+{
+        struct tegra_drm_file *fpriv = file->driver_priv;
+        struct drm_tegra_get_syncpt_base *args = data;
+        struct tegra_drm_context *context;
+        struct host1x_syncpt_base *base;
+        struct host1x_syncpt *syncpt;
+
+        context = tegra_drm_get_context(args->context);
+
+        if (!tegra_drm_file_owns_context(fpriv, context))
+                return -ENODEV;
+
+        if (args->syncpt >= context->client->base.num_syncpts)
+                return -EINVAL;
+
+        syncpt = context->client->base.syncpts[args->syncpt];
+
+        base = host1x_syncpt_get_base(syncpt);
+        if (!base)
+                return -ENXIO;
+
+        args->id = host1x_syncpt_base_id(base);
+
+        return 0;
+}
 #endif
 
 static const struct drm_ioctl_desc tegra_drm_ioctls[] = {
@@ -498,6 +459,7 @@ static const struct drm_ioctl_desc tegra_drm_ioctls[] = {
         DRM_IOCTL_DEF_DRV(TEGRA_CLOSE_CHANNEL, tegra_close_channel, DRM_UNLOCKED),
         DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT, tegra_get_syncpt, DRM_UNLOCKED),
         DRM_IOCTL_DEF_DRV(TEGRA_SUBMIT, tegra_submit, DRM_UNLOCKED),
+        DRM_IOCTL_DEF_DRV(TEGRA_GET_SYNCPT_BASE, tegra_get_syncpt_base, DRM_UNLOCKED),
 #endif
 };
 
@@ -559,15 +521,15 @@ static void tegra_drm_disable_vblank(struct drm_device *drm, int pipe)
 
 static void tegra_drm_preclose(struct drm_device *drm, struct drm_file *file)
 {
-        struct host1x_drm_file *fpriv = file->driver_priv;
-        struct host1x_drm_context *context, *tmp;
+        struct tegra_drm_file *fpriv = file->driver_priv;
+        struct tegra_drm_context *context, *tmp;
         struct drm_crtc *crtc;
 
         list_for_each_entry(crtc, &drm->mode_config.crtc_list, head)
                 tegra_dc_cancel_page_flip(crtc, file);
 
         list_for_each_entry_safe(context, tmp, &fpriv->contexts, list)
-                host1x_drm_context_free(context);
+                tegra_drm_context_free(context);
 
         kfree(fpriv);
 }
@@ -645,3 +607,108 @@ struct drm_driver tegra_drm_driver = {
         .minor = DRIVER_MINOR,
         .patchlevel = DRIVER_PATCHLEVEL,
 };
+
+int tegra_drm_register_client(struct tegra_drm *tegra,
+                              struct tegra_drm_client *client)
+{
+        mutex_lock(&tegra->clients_lock);
+        list_add_tail(&client->list, &tegra->clients);
+        mutex_unlock(&tegra->clients_lock);
+
+        return 0;
+}
+
+int tegra_drm_unregister_client(struct tegra_drm *tegra,
+                                struct tegra_drm_client *client)
+{
+        mutex_lock(&tegra->clients_lock);
+        list_del_init(&client->list);
+        mutex_unlock(&tegra->clients_lock);
+
+        return 0;
+}
+
+static int host1x_drm_probe(struct host1x_device *device)
+{
+        return drm_host1x_init(&tegra_drm_driver, device);
+}
+
+static int host1x_drm_remove(struct host1x_device *device)
+{
+        drm_host1x_exit(&tegra_drm_driver, device);
+
+        return 0;
+}
+
+static const struct of_device_id host1x_drm_subdevs[] = {
+        { .compatible = "nvidia,tegra20-dc", },
+        { .compatible = "nvidia,tegra20-hdmi", },
+        { .compatible = "nvidia,tegra20-gr2d", },
+        { .compatible = "nvidia,tegra20-gr3d", },
+        { .compatible = "nvidia,tegra30-dc", },
+        { .compatible = "nvidia,tegra30-hdmi", },
+        { .compatible = "nvidia,tegra30-gr2d", },
+        { .compatible = "nvidia,tegra30-gr3d", },
+        { .compatible = "nvidia,tegra114-hdmi", },
+        { .compatible = "nvidia,tegra114-gr3d", },
+        { /* sentinel */ }
+};
+
+static struct host1x_driver host1x_drm_driver = {
+        .name = "drm",
+        .probe = host1x_drm_probe,
+        .remove = host1x_drm_remove,
+        .subdevs = host1x_drm_subdevs,
+};
+
+static int __init host1x_drm_init(void)
+{
+        int err;
+
+        err = host1x_driver_register(&host1x_drm_driver);
+        if (err < 0)
+                return err;
+
+        err = platform_driver_register(&tegra_dc_driver);
+        if (err < 0)
+                goto unregister_host1x;
+
+        err = platform_driver_register(&tegra_hdmi_driver);
+        if (err < 0)
+                goto unregister_dc;
+
+        err = platform_driver_register(&tegra_gr2d_driver);
+        if (err < 0)
+                goto unregister_hdmi;
+
+        err = platform_driver_register(&tegra_gr3d_driver);
+        if (err < 0)
+                goto unregister_gr2d;
+
+        return 0;
+
+unregister_gr2d:
+        platform_driver_unregister(&tegra_gr2d_driver);
+unregister_hdmi:
+        platform_driver_unregister(&tegra_hdmi_driver);
+unregister_dc:
+        platform_driver_unregister(&tegra_dc_driver);
+unregister_host1x:
+        host1x_driver_unregister(&host1x_drm_driver);
+        return err;
+}
+module_init(host1x_drm_init);
+
+static void __exit host1x_drm_exit(void)
+{
+        platform_driver_unregister(&tegra_gr3d_driver);
+        platform_driver_unregister(&tegra_gr2d_driver);
+        platform_driver_unregister(&tegra_hdmi_driver);
+        platform_driver_unregister(&tegra_dc_driver);
+        host1x_driver_unregister(&host1x_drm_driver);
+}
+module_exit(host1x_drm_exit);
+
+MODULE_AUTHOR("Thierry Reding <thierry.reding@avionic-design.de>");
+MODULE_DESCRIPTION("NVIDIA Tegra DRM driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/host1x/drm/drm.h b/drivers/gpu/drm/tegra/drm.h
index 02ce020f2575..fdfe259ed7f8 100644
--- a/drivers/gpu/host1x/drm/drm.h
+++ b/drivers/gpu/drm/tegra/drm.h
@@ -10,14 +10,14 @@
 #ifndef HOST1X_DRM_H
 #define HOST1X_DRM_H 1
 
+#include <uapi/drm/tegra_drm.h>
+#include <linux/host1x.h>
+
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_fixed.h>
-#include <uapi/drm/tegra_drm.h>
-
-#include "host1x.h"
 
 struct tegra_fb {
         struct drm_framebuffer base;
@@ -30,17 +30,8 @@ struct tegra_fbdev {
         struct tegra_fb *fb;
 };
 
-struct host1x_drm {
+struct tegra_drm {
         struct drm_device *drm;
-        struct device *dev;
-        void __iomem *regs;
-        struct clk *clk;
-        int syncpt;
-        int irq;
-
-        struct mutex drm_clients_lock;
-        struct list_head drm_clients;
-        struct list_head drm_active;
 
         struct mutex clients_lock;
         struct list_head clients;
@@ -48,66 +39,60 @@ struct host1x_drm {
         struct tegra_fbdev *fbdev;
 };
 
-struct host1x_client;
+struct tegra_drm_client;
 
-struct host1x_drm_context {
-        struct host1x_client *client;
+struct tegra_drm_context {
+        struct tegra_drm_client *client;
         struct host1x_channel *channel;
         struct list_head list;
 };
 
-struct host1x_client_ops {
-        int (*drm_init)(struct host1x_client *client, struct drm_device *drm);
-        int (*drm_exit)(struct host1x_client *client);
-        int (*open_channel)(struct host1x_client *client,
-                            struct host1x_drm_context *context);
-        void (*close_channel)(struct host1x_drm_context *context);
-        int (*submit)(struct host1x_drm_context *context,
+struct tegra_drm_client_ops {
+        int (*open_channel)(struct tegra_drm_client *client,
+                            struct tegra_drm_context *context);
+        void (*close_channel)(struct tegra_drm_context *context);
+        int (*is_addr_reg)(struct device *dev, u32 class, u32 offset);
+        int (*submit)(struct tegra_drm_context *context,
                       struct drm_tegra_submit *args, struct drm_device *drm,
                       struct drm_file *file);
 };
 
-struct host1x_drm_file {
-        struct list_head contexts;
-};
-
-struct host1x_client {
-        struct host1x_drm *host1x;
-        struct device *dev;
-
-        const struct host1x_client_ops *ops;
-
-        enum host1x_class class;
-        struct host1x_channel *channel;
-
-        struct host1x_syncpt **syncpts;
-        unsigned int num_syncpts;
+int tegra_drm_submit(struct tegra_drm_context *context,
+                     struct drm_tegra_submit *args, struct drm_device *drm,
+                     struct drm_file *file);
 
+struct tegra_drm_client {
+        struct host1x_client base;
         struct list_head list;
+
+        const struct tegra_drm_client_ops *ops;
 };
 
-extern int host1x_drm_init(struct host1x_drm *host1x, struct drm_device *drm);
-extern int host1x_drm_exit(struct host1x_drm *host1x);
+static inline struct tegra_drm_client *
+host1x_to_drm_client(struct host1x_client *client)
+{
+        return container_of(client, struct tegra_drm_client, base);
+}
+
+extern int tegra_drm_register_client(struct tegra_drm *tegra,
+                                     struct tegra_drm_client *client);
+extern int tegra_drm_unregister_client(struct tegra_drm *tegra,
+                                       struct tegra_drm_client *client);
 
-extern int host1x_register_client(struct host1x_drm *host1x,
-                                  struct host1x_client *client);
-extern int host1x_unregister_client(struct host1x_drm *host1x,
-                                    struct host1x_client *client);
+extern int tegra_drm_init(struct tegra_drm *tegra, struct drm_device *drm);
+extern int tegra_drm_exit(struct tegra_drm *tegra);
 
 struct tegra_output;
 
 struct tegra_dc {
         struct host1x_client client;
-        spinlock_t lock;
-
-        struct host1x_drm *host1x;
         struct device *dev;
+        spinlock_t lock;
 
         struct drm_crtc base;
         int pipe;
 
         struct clk *clk;
-
         void __iomem *regs;
         int irq;
 
@@ -123,7 +108,8 @@ struct tegra_dc {
         struct drm_pending_vblank_event *event;
 };
 
-static inline struct tegra_dc *host1x_client_to_dc(struct host1x_client *client)
+static inline struct tegra_dc *
+host1x_client_to_dc(struct host1x_client *client)
 {
         return container_of(client, struct tegra_dc, client);
 }
@@ -162,6 +148,8 @@ struct tegra_dc_window {
         unsigned int format;
         unsigned int stride[2];
         unsigned long base[3];
+        bool bottom_up;
+        bool tiled;
 };
 
 /* from dc.c */
@@ -249,23 +237,34 @@ static inline int tegra_output_check_mode(struct tegra_output *output,
         return output ? -ENOSYS : -EINVAL;
 }
 
+/* from bus.c */
+int drm_host1x_init(struct drm_driver *driver, struct host1x_device *device);
+void drm_host1x_exit(struct drm_driver *driver, struct host1x_device *device);
+
 /* from rgb.c */
 extern int tegra_dc_rgb_probe(struct tegra_dc *dc);
+extern int tegra_dc_rgb_remove(struct tegra_dc *dc);
 extern int tegra_dc_rgb_init(struct drm_device *drm, struct tegra_dc *dc);
 extern int tegra_dc_rgb_exit(struct tegra_dc *dc);
 
 /* from output.c */
-extern int tegra_output_parse_dt(struct tegra_output *output);
+extern int tegra_output_probe(struct tegra_output *output);
+extern int tegra_output_remove(struct tegra_output *output);
 extern int tegra_output_init(struct drm_device *drm, struct tegra_output *output);
 extern int tegra_output_exit(struct tegra_output *output);
 
 /* from fb.c */
 struct tegra_bo *tegra_fb_get_plane(struct drm_framebuffer *framebuffer,
                                     unsigned int index);
+bool tegra_fb_is_bottom_up(struct drm_framebuffer *framebuffer);
+bool tegra_fb_is_tiled(struct drm_framebuffer *framebuffer);
 extern int tegra_drm_fb_init(struct drm_device *drm);
 extern void tegra_drm_fb_exit(struct drm_device *drm);
 extern void tegra_fbdev_restore_mode(struct tegra_fbdev *fbdev);
 
-extern struct drm_driver tegra_drm_driver;
+extern struct platform_driver tegra_dc_driver;
+extern struct platform_driver tegra_hdmi_driver;
+extern struct platform_driver tegra_gr2d_driver;
+extern struct platform_driver tegra_gr3d_driver;
 
 #endif /* HOST1X_DRM_H */
diff --git a/drivers/gpu/host1x/drm/fb.c b/drivers/gpu/drm/tegra/fb.c
index 979a3e32b78b..490f7719e317 100644
--- a/drivers/gpu/host1x/drm/fb.c
+++ b/drivers/gpu/drm/tegra/fb.c
@@ -10,8 +10,6 @@
  * published by the Free Software Foundation.
  */
 
-#include <linux/module.h>
-
 #include "drm.h"
 #include "gem.h"
 
@@ -36,6 +34,26 @@ struct tegra_bo *tegra_fb_get_plane(struct drm_framebuffer *framebuffer,
         return fb->planes[index];
 }
 
+bool tegra_fb_is_bottom_up(struct drm_framebuffer *framebuffer)
+{
+        struct tegra_fb *fb = to_tegra_fb(framebuffer);
+
+        if (fb->planes[0]->flags & TEGRA_BO_BOTTOM_UP)
+                return true;
+
+        return false;
+}
+
+bool tegra_fb_is_tiled(struct drm_framebuffer *framebuffer)
+{
+        struct tegra_fb *fb = to_tegra_fb(framebuffer);
+
+        if (fb->planes[0]->flags & TEGRA_BO_TILED)
+                return true;
+
+        return false;
+}
+
 static void tegra_fb_destroy(struct drm_framebuffer *framebuffer)
 {
         struct tegra_fb *fb = to_tegra_fb(framebuffer);
@@ -190,7 +208,7 @@ static int tegra_fbdev_probe(struct drm_fb_helper *helper,
 
         size = cmd.pitches[0] * cmd.height;
 
-        bo = tegra_bo_create(drm, size);
+        bo = tegra_bo_create(drm, size, 0);
         if (IS_ERR(bo))
                 return PTR_ERR(bo);
 
@@ -323,10 +341,10 @@ static void tegra_fbdev_free(struct tegra_fbdev *fbdev)
 
 static void tegra_fb_output_poll_changed(struct drm_device *drm)
 {
-        struct host1x_drm *host1x = drm->dev_private;
+        struct tegra_drm *tegra = drm->dev_private;
 
-        if (host1x->fbdev)
-                drm_fb_helper_hotplug_event(&host1x->fbdev->base);
+        if (tegra->fbdev)
+                drm_fb_helper_hotplug_event(&tegra->fbdev->base);
 }
 
 static const struct drm_mode_config_funcs tegra_drm_mode_funcs = {
@@ -336,7 +354,7 @@ static const struct drm_mode_config_funcs tegra_drm_mode_funcs = {
 
 int tegra_drm_fb_init(struct drm_device *drm)
 {
-        struct host1x_drm *host1x = drm->dev_private;
+        struct tegra_drm *tegra = drm->dev_private;
         struct tegra_fbdev *fbdev;
 
         drm->mode_config.min_width = 0;
@@ -352,16 +370,16 @@ int tegra_drm_fb_init(struct drm_device *drm)
         if (IS_ERR(fbdev))
                 return PTR_ERR(fbdev);
 
-        host1x->fbdev = fbdev;
+        tegra->fbdev = fbdev;
 
         return 0;
 }
 
 void tegra_drm_fb_exit(struct drm_device *drm)
 {
-        struct host1x_drm *host1x = drm->dev_private;
+        struct tegra_drm *tegra = drm->dev_private;
 
-        tegra_fbdev_free(host1x->fbdev);
+        tegra_fbdev_free(tegra->fbdev);
 }
 
 void tegra_fbdev_restore_mode(struct tegra_fbdev *fbdev)
diff --git a/drivers/gpu/host1x/drm/gem.c b/drivers/gpu/drm/tegra/gem.c
index 59623de4ee15..28a9cbc07ab9 100644
--- a/drivers/gpu/host1x/drm/gem.c
+++ b/drivers/gpu/drm/tegra/gem.c
@@ -18,25 +18,18 @@
  * GNU General Public License for more details.
  */
 
-#include <linux/mm.h>
-#include <linux/slab.h>
-#include <linux/mutex.h>
-#include <linux/export.h>
-#include <linux/dma-mapping.h>
-
-#include <drm/drmP.h>
-#include <drm/drm.h>
+#include <drm/tegra_drm.h>
 
 #include "gem.h"
 
-static inline struct tegra_bo *host1x_to_drm_bo(struct host1x_bo *bo)
+static inline struct tegra_bo *host1x_to_tegra_bo(struct host1x_bo *bo)
 {
         return container_of(bo, struct tegra_bo, base);
 }
 
 static void tegra_bo_put(struct host1x_bo *bo)
 {
-        struct tegra_bo *obj = host1x_to_drm_bo(bo);
+        struct tegra_bo *obj = host1x_to_tegra_bo(bo);
         struct drm_device *drm = obj->gem.dev;
 
         mutex_lock(&drm->struct_mutex);
@@ -46,7 +39,7 @@ static void tegra_bo_put(struct host1x_bo *bo)
 
 static dma_addr_t tegra_bo_pin(struct host1x_bo *bo, struct sg_table **sgt)
 {
-        struct tegra_bo *obj = host1x_to_drm_bo(bo);
+        struct tegra_bo *obj = host1x_to_tegra_bo(bo);
 
         return obj->paddr;
 }
@@ -57,7 +50,7 @@ static void tegra_bo_unpin(struct host1x_bo *bo, struct sg_table *sgt)
 
 static void *tegra_bo_mmap(struct host1x_bo *bo)
 {
-        struct tegra_bo *obj = host1x_to_drm_bo(bo);
+        struct tegra_bo *obj = host1x_to_tegra_bo(bo);
 
         return obj->vaddr;
 }
@@ -68,7 +61,7 @@ static void tegra_bo_munmap(struct host1x_bo *bo, void *addr)
 
 static void *tegra_bo_kmap(struct host1x_bo *bo, unsigned int page)
 {
-        struct tegra_bo *obj = host1x_to_drm_bo(bo);
+        struct tegra_bo *obj = host1x_to_tegra_bo(bo);
 
         return obj->vaddr + page * PAGE_SIZE;
 }
@@ -80,7 +73,7 @@ static void tegra_bo_kunmap(struct host1x_bo *bo, unsigned int page,
 
 static struct host1x_bo *tegra_bo_get(struct host1x_bo *bo)
 {
-        struct tegra_bo *obj = host1x_to_drm_bo(bo);
+        struct tegra_bo *obj = host1x_to_tegra_bo(bo);
         struct drm_device *drm = obj->gem.dev;
 
         mutex_lock(&drm->struct_mutex);
@@ -106,7 +99,8 @@ static void tegra_bo_destroy(struct drm_device *drm, struct tegra_bo *bo)
         dma_free_writecombine(drm->dev, bo->gem.size, bo->vaddr, bo->paddr);
 }
 
-struct tegra_bo *tegra_bo_create(struct drm_device *drm, unsigned int size)
+struct tegra_bo *tegra_bo_create(struct drm_device *drm, unsigned int size,
+                                 unsigned long flags)
 {
         struct tegra_bo *bo;
         int err;
@@ -135,6 +129,12 @@ struct tegra_bo *tegra_bo_create(struct drm_device *drm, unsigned int size)
         if (err)
                 goto err_mmap;
 
+        if (flags & DRM_TEGRA_GEM_CREATE_TILED)
+                bo->flags |= TEGRA_BO_TILED;
+
+        if (flags & DRM_TEGRA_GEM_CREATE_BOTTOM_UP)
+                bo->flags |= TEGRA_BO_BOTTOM_UP;
+
         return bo;
 
 err_mmap:
@@ -149,14 +149,15 @@ err_dma:
 }
 
 struct tegra_bo *tegra_bo_create_with_handle(struct drm_file *file,
-                                            struct drm_device *drm,
-                                            unsigned int size,
-                                            unsigned int *handle)
+                                             struct drm_device *drm,
+                                             unsigned int size,
+                                             unsigned long flags,
+                                             unsigned int *handle)
 {
         struct tegra_bo *bo;
         int ret;
 
-        bo = tegra_bo_create(drm, size);
+        bo = tegra_bo_create(drm, size, flags);
         if (IS_ERR(bo))
                 return bo;
 
@@ -178,7 +179,6 @@ void tegra_bo_free_object(struct drm_gem_object *gem)
         struct tegra_bo *bo = to_tegra_bo(gem);
 
         drm_gem_free_mmap_offset(gem);
-
         drm_gem_object_release(gem);
         tegra_bo_destroy(gem->dev, bo);
 
@@ -197,8 +197,8 @@ int tegra_bo_dumb_create(struct drm_file *file, struct drm_device *drm,
         if (args->size < args->pitch * args->height)
                 args->size = args->pitch * args->height;
 
-        bo = tegra_bo_create_with_handle(file, drm, args->size,
-                                            &args->handle);
+        bo = tegra_bo_create_with_handle(file, drm, args->size, 0,
+                                         &args->handle);
         if (IS_ERR(bo))
                 return PTR_ERR(bo);
 
diff --git a/drivers/gpu/host1x/drm/gem.h b/drivers/gpu/drm/tegra/gem.h
index 492533a2dacb..7674000bf47d 100644
--- a/drivers/gpu/host1x/drm/gem.h
+++ b/drivers/gpu/drm/tegra/gem.h
@@ -19,14 +19,18 @@
 #ifndef __HOST1X_GEM_H
 #define __HOST1X_GEM_H
 
+#include <linux/host1x.h>
+
 #include <drm/drm.h>
 #include <drm/drmP.h>
 
-#include "host1x_bo.h"
+#define TEGRA_BO_TILED     (1 << 0)
+#define TEGRA_BO_BOTTOM_UP (1 << 1)
 
 struct tegra_bo {
         struct drm_gem_object gem;
         struct host1x_bo base;
+        unsigned long flags;
         dma_addr_t paddr;
         void *vaddr;
 };
@@ -38,11 +42,13 @@ static inline struct tegra_bo *to_tegra_bo(struct drm_gem_object *gem)
 
 extern const struct host1x_bo_ops tegra_bo_ops;
 
-struct tegra_bo *tegra_bo_create(struct drm_device *drm, unsigned int size);
+struct tegra_bo *tegra_bo_create(struct drm_device *drm, unsigned int size,
+                                 unsigned long flags);
 struct tegra_bo *tegra_bo_create_with_handle(struct drm_file *file,
-                                            struct drm_device *drm,
-                                            unsigned int size,
-                                            unsigned int *handle);
+                                             struct drm_device *drm,
+                                             unsigned int size,
+                                             unsigned long flags,
+                                             unsigned int *handle);
 void tegra_bo_free_object(struct drm_gem_object *gem);
 int tegra_bo_dumb_create(struct drm_file *file, struct drm_device *drm,
                          struct drm_mode_create_dumb *args);
diff --git a/drivers/gpu/drm/tegra/gr2d.c b/drivers/gpu/drm/tegra/gr2d.c
new file mode 100644
index 000000000000..7ec4259ffded
--- /dev/null
+++ b/drivers/gpu/drm/tegra/gr2d.c
@@ -0,0 +1,227 @@
+/*
+ * Copyright (c) 2012-2013, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/clk.h>
+
+#include "drm.h"
+#include "gem.h"
+#include "gr2d.h"
+
+struct gr2d {
+        struct tegra_drm_client client;
+        struct host1x_channel *channel;
+        struct clk *clk;
+
+        DECLARE_BITMAP(addr_regs, GR2D_NUM_REGS);
+};
+
+static inline struct gr2d *to_gr2d(struct tegra_drm_client *client)
+{
+        return container_of(client, struct gr2d, client);
+}
+
+static int gr2d_init(struct host1x_client *client)
+{
+        struct tegra_drm_client *drm = host1x_to_drm_client(client);
+        struct tegra_drm *tegra = dev_get_drvdata(client->parent);
+        unsigned long flags = HOST1X_SYNCPT_HAS_BASE;
+        struct gr2d *gr2d = to_gr2d(drm);
+
+        gr2d->channel = host1x_channel_request(client->dev);
+        if (!gr2d->channel)
+                return -ENOMEM;
+
+        client->syncpts[0] = host1x_syncpt_request(client->dev, flags);
+        if (!client->syncpts[0]) {
+                host1x_channel_free(gr2d->channel);
+                return -ENOMEM;
+        }
+
+        return tegra_drm_register_client(tegra, drm);
+}
+
+static int gr2d_exit(struct host1x_client *client)
+{
+        struct tegra_drm_client *drm = host1x_to_drm_client(client);
+        struct tegra_drm *tegra = dev_get_drvdata(client->parent);
+        struct gr2d *gr2d = to_gr2d(drm);
+        int err;
+
+        err = tegra_drm_unregister_client(tegra, drm);
+        if (err < 0)
+                return err;
+
+        host1x_syncpt_free(client->syncpts[0]);
+        host1x_channel_free(gr2d->channel);
+
+        return 0;
+}
+
+static const struct host1x_client_ops gr2d_client_ops = {
+        .init = gr2d_init,
+        .exit = gr2d_exit,
+};
+
+static int gr2d_open_channel(struct tegra_drm_client *client,
+                             struct tegra_drm_context *context)
+{
+        struct gr2d *gr2d = to_gr2d(client);
+
+        context->channel = host1x_channel_get(gr2d->channel);
+        if (!context->channel)
+                return -ENOMEM;
+
+        return 0;
+}
+
+static void gr2d_close_channel(struct tegra_drm_context *context)
+{
+        host1x_channel_put(context->channel);
+}
+
+static int gr2d_is_addr_reg(struct device *dev, u32 class, u32 offset)
+{
+        struct gr2d *gr2d = dev_get_drvdata(dev);
+
+        switch (class) {
+        case HOST1X_CLASS_HOST1X:
+                if (offset == 0x2b)
+                        return 1;
+
+                break;
+
+        case HOST1X_CLASS_GR2D:
+        case HOST1X_CLASS_GR2D_SB:
+                if (offset >= GR2D_NUM_REGS)
+                        break;
+
+                if (test_bit(offset, gr2d->addr_regs))
+                        return 1;
+
+                break;
+        }
+
+        return 0;
+}
+
+static const struct tegra_drm_client_ops gr2d_ops = {
+        .open_channel = gr2d_open_channel,
+        .close_channel = gr2d_close_channel,
+        .is_addr_reg = gr2d_is_addr_reg,
+        .submit = tegra_drm_submit,
+};
+
+static const struct of_device_id gr2d_match[] = {
+        { .compatible = "nvidia,tegra30-gr2d" },
+        { .compatible = "nvidia,tegra20-gr2d" },
+        { },
+};
+
+static const u32 gr2d_addr_regs[] = {
+        GR2D_UA_BASE_ADDR,
+        GR2D_VA_BASE_ADDR,
+        GR2D_PAT_BASE_ADDR,
+        GR2D_DSTA_BASE_ADDR,
+        GR2D_DSTB_BASE_ADDR,
+        GR2D_DSTC_BASE_ADDR,
+        GR2D_SRCA_BASE_ADDR,
+        GR2D_SRCB_BASE_ADDR,
+        GR2D_SRC_BASE_ADDR_SB,
+        GR2D_DSTA_BASE_ADDR_SB,
+        GR2D_DSTB_BASE_ADDR_SB,
+        GR2D_UA_BASE_ADDR_SB,
+        GR2D_VA_BASE_ADDR_SB,
+};
+
+static int gr2d_probe(struct platform_device *pdev)
+{
+        struct device *dev = &pdev->dev;
+        struct host1x_syncpt **syncpts;
+        struct gr2d *gr2d;
+        unsigned int i;
+        int err;
+
+        gr2d = devm_kzalloc(dev, sizeof(*gr2d), GFP_KERNEL);
+        if (!gr2d)
+                return -ENOMEM;
+
+        syncpts = devm_kzalloc(dev, sizeof(*syncpts), GFP_KERNEL);
+        if (!syncpts)
+                return -ENOMEM;
+
+        gr2d->clk = devm_clk_get(dev, NULL);
+        if (IS_ERR(gr2d->clk)) {
+                dev_err(dev, "cannot get clock\n");
+                return PTR_ERR(gr2d->clk);
+        }
+
+        err = clk_prepare_enable(gr2d->clk);
+        if (err) {
+                dev_err(dev, "cannot turn on clock\n");
+                return err;
+        }
+
+        INIT_LIST_HEAD(&gr2d->client.base.list);
+        gr2d->client.base.ops = &gr2d_client_ops;
+        gr2d->client.base.dev = dev;
+        gr2d->client.base.class = HOST1X_CLASS_GR2D;
+        gr2d->client.base.syncpts = syncpts;
+        gr2d->client.base.num_syncpts = 1;
+
+        INIT_LIST_HEAD(&gr2d->client.list);
+        gr2d->client.ops = &gr2d_ops;
+
+        err = host1x_client_register(&gr2d->client.base);
+        if (err < 0) {
+                dev_err(dev, "failed to register host1x client: %d\n", err);
+                clk_disable_unprepare(gr2d->clk);
+                return err;
+        }
+
+        /* initialize address register map */
+        for (i = 0; i < ARRAY_SIZE(gr2d_addr_regs); i++)
+                set_bit(gr2d_addr_regs[i], gr2d->addr_regs);
+
+        platform_set_drvdata(pdev, gr2d);
+
+        return 0;
+}
+
+static int gr2d_remove(struct platform_device *pdev)
+{
+        struct gr2d *gr2d = platform_get_drvdata(pdev);
+        int err;
+
+        err = host1x_client_unregister(&gr2d->client.base);
+        if (err < 0) {
+                dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
+                        err);
+                return err;
+        }
+
+        clk_disable_unprepare(gr2d->clk);
+
+        return 0;
+}
+
+struct platform_driver tegra_gr2d_driver = {
+        .driver = {
+                .name = "tegra-gr2d",
+                .of_match_table = gr2d_match,
+        },
+        .probe = gr2d_probe,
+        .remove = gr2d_remove,
+};
diff --git a/drivers/gpu/drm/tegra/gr2d.h b/drivers/gpu/drm/tegra/gr2d.h
new file mode 100644
index 000000000000..4d7304fb015e
--- /dev/null
+++ b/drivers/gpu/drm/tegra/gr2d.h
@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2013 NVIDIA Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef TEGRA_GR2D_H
+#define TEGRA_GR2D_H
+
+#define GR2D_UA_BASE_ADDR               0x1a
+#define GR2D_VA_BASE_ADDR               0x1b
+#define GR2D_PAT_BASE_ADDR              0x26
+#define GR2D_DSTA_BASE_ADDR             0x2b
+#define GR2D_DSTB_BASE_ADDR             0x2c
+#define GR2D_DSTC_BASE_ADDR             0x2d
+#define GR2D_SRCA_BASE_ADDR             0x31
+#define GR2D_SRCB_BASE_ADDR             0x32
+#define GR2D_SRC_BASE_ADDR_SB           0x48
+#define GR2D_DSTA_BASE_ADDR_SB          0x49
+#define GR2D_DSTB_BASE_ADDR_SB          0x4a
+#define GR2D_UA_BASE_ADDR_SB            0x4b
+#define GR2D_VA_BASE_ADDR_SB            0x4c
+
+#define GR2D_NUM_REGS                   0x4d
+
+#endif
diff --git a/drivers/gpu/drm/tegra/gr3d.c b/drivers/gpu/drm/tegra/gr3d.c
new file mode 100644
index 000000000000..4cec8f526af7
--- /dev/null
+++ b/drivers/gpu/drm/tegra/gr3d.c
@@ -0,0 +1,338 @@
+/*
+ * Copyright (C) 2013 Avionic Design GmbH
+ * Copyright (C) 2013 NVIDIA Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/clk.h>
+#include <linux/host1x.h>
+#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/tegra-powergate.h>
+
+#include "drm.h"
+#include "gem.h"
+#include "gr3d.h"
+
+struct gr3d {
+        struct tegra_drm_client client;
+        struct host1x_channel *channel;
+        struct clk *clk_secondary;
+        struct clk *clk;
+
+        DECLARE_BITMAP(addr_regs, GR3D_NUM_REGS);
+};
+
+static inline struct gr3d *to_gr3d(struct tegra_drm_client *client)
+{
+        return container_of(client, struct gr3d, client);
+}
+
+static int gr3d_init(struct host1x_client *client)
+{
+        struct tegra_drm_client *drm = host1x_to_drm_client(client);
+        struct tegra_drm *tegra = dev_get_drvdata(client->parent);
+        unsigned long flags = HOST1X_SYNCPT_HAS_BASE;
+        struct gr3d *gr3d = to_gr3d(drm);
+
+        gr3d->channel = host1x_channel_request(client->dev);
+        if (!gr3d->channel)
+                return -ENOMEM;
+
+        client->syncpts[0] = host1x_syncpt_request(client->dev, flags);
+        if (!client->syncpts[0]) {
+                host1x_channel_free(gr3d->channel);
+                return -ENOMEM;
+        }
+
+        return tegra_drm_register_client(tegra, drm);
+}
+
+static int gr3d_exit(struct host1x_client *client)
+{
+        struct tegra_drm_client *drm = host1x_to_drm_client(client);
+        struct tegra_drm *tegra = dev_get_drvdata(client->parent);
+        struct gr3d *gr3d = to_gr3d(drm);
+        int err;
+
+        err = tegra_drm_unregister_client(tegra, drm);
+        if (err < 0)
+                return err;
+
+        host1x_syncpt_free(client->syncpts[0]);
+        host1x_channel_free(gr3d->channel);
+
+        return 0;
+}
+
+static const struct host1x_client_ops gr3d_client_ops = {
+        .init = gr3d_init,
+        .exit = gr3d_exit,
+};
+
+static int gr3d_open_channel(struct tegra_drm_client *client,
+                             struct tegra_drm_context *context)
+{
+        struct gr3d *gr3d = to_gr3d(client);
+
+        context->channel = host1x_channel_get(gr3d->channel);
+        if (!context->channel)
+                return -ENOMEM;
+
+        return 0;
+}
+
+static void gr3d_close_channel(struct tegra_drm_context *context)
+{
+        host1x_channel_put(context->channel);
+}
+
+static int gr3d_is_addr_reg(struct device *dev, u32 class, u32 offset)
+{
+        struct gr3d *gr3d = dev_get_drvdata(dev);
+
+        switch (class) {
+        case HOST1X_CLASS_HOST1X:
+                if (offset == 0x2b)
+                        return 1;
+
+                break;
+
+        case HOST1X_CLASS_GR3D:
+                if (offset >= GR3D_NUM_REGS)
+                        break;
+
+                if (test_bit(offset, gr3d->addr_regs))
+                        return 1;
+
+                break;
+        }
+
+        return 0;
+}
+
+static const struct tegra_drm_client_ops gr3d_ops = {
+        .open_channel = gr3d_open_channel,
+        .close_channel = gr3d_close_channel,
+        .is_addr_reg = gr3d_is_addr_reg,
+        .submit = tegra_drm_submit,
+};
+
+static const struct of_device_id tegra_gr3d_match[] = {
+        { .compatible = "nvidia,tegra114-gr3d" },
+        { .compatible = "nvidia,tegra30-gr3d" },
+        { .compatible = "nvidia,tegra20-gr3d" },
+        { }
+};
+
+static const u32 gr3d_addr_regs[] = {
+        GR3D_IDX_ATTRIBUTE( 0),
+        GR3D_IDX_ATTRIBUTE( 1),
+        GR3D_IDX_ATTRIBUTE( 2),
+        GR3D_IDX_ATTRIBUTE( 3),
+        GR3D_IDX_ATTRIBUTE( 4),
+        GR3D_IDX_ATTRIBUTE( 5),
+        GR3D_IDX_ATTRIBUTE( 6),
+        GR3D_IDX_ATTRIBUTE( 7),
+        GR3D_IDX_ATTRIBUTE( 8),
+        GR3D_IDX_ATTRIBUTE( 9),
+        GR3D_IDX_ATTRIBUTE(10),
+        GR3D_IDX_ATTRIBUTE(11),
+        GR3D_IDX_ATTRIBUTE(12),
+        GR3D_IDX_ATTRIBUTE(13),
+        GR3D_IDX_ATTRIBUTE(14),
+        GR3D_IDX_ATTRIBUTE(15),
+        GR3D_IDX_INDEX_BASE,
+        GR3D_QR_ZTAG_ADDR,
+        GR3D_QR_CTAG_ADDR,
+        GR3D_QR_CZ_ADDR,
+        GR3D_TEX_TEX_ADDR( 0),
+        GR3D_TEX_TEX_ADDR( 1),
+        GR3D_TEX_TEX_ADDR( 2),
+        GR3D_TEX_TEX_ADDR( 3),
+        GR3D_TEX_TEX_ADDR( 4),
+        GR3D_TEX_TEX_ADDR( 5),
+        GR3D_TEX_TEX_ADDR( 6),
+        GR3D_TEX_TEX_ADDR( 7),
+        GR3D_TEX_TEX_ADDR( 8),
+        GR3D_TEX_TEX_ADDR( 9),
+        GR3D_TEX_TEX_ADDR(10),
+        GR3D_TEX_TEX_ADDR(11),
+        GR3D_TEX_TEX_ADDR(12),
+        GR3D_TEX_TEX_ADDR(13),
+        GR3D_TEX_TEX_ADDR(14),
+        GR3D_TEX_TEX_ADDR(15),
+        GR3D_DW_MEMORY_OUTPUT_ADDRESS,
+        GR3D_GLOBAL_SURFADDR( 0),
+        GR3D_GLOBAL_SURFADDR( 1),
+        GR3D_GLOBAL_SURFADDR( 2),
+        GR3D_GLOBAL_SURFADDR( 3),
+        GR3D_GLOBAL_SURFADDR( 4),
+        GR3D_GLOBAL_SURFADDR( 5),
+        GR3D_GLOBAL_SURFADDR( 6),
+        GR3D_GLOBAL_SURFADDR( 7),
+        GR3D_GLOBAL_SURFADDR( 8),
+        GR3D_GLOBAL_SURFADDR( 9),
+        GR3D_GLOBAL_SURFADDR(10),
+        GR3D_GLOBAL_SURFADDR(11),
+        GR3D_GLOBAL_SURFADDR(12),
+        GR3D_GLOBAL_SURFADDR(13),
+        GR3D_GLOBAL_SURFADDR(14),
+        GR3D_GLOBAL_SURFADDR(15),
+        GR3D_GLOBAL_SPILLSURFADDR,
+        GR3D_GLOBAL_SURFOVERADDR( 0),
+        GR3D_GLOBAL_SURFOVERADDR( 1),
+        GR3D_GLOBAL_SURFOVERADDR( 2),
+        GR3D_GLOBAL_SURFOVERADDR( 3),
+        GR3D_GLOBAL_SURFOVERADDR( 4),
+        GR3D_GLOBAL_SURFOVERADDR( 5),
+        GR3D_GLOBAL_SURFOVERADDR( 6),
+        GR3D_GLOBAL_SURFOVERADDR( 7),
+        GR3D_GLOBAL_SURFOVERADDR( 8),
+        GR3D_GLOBAL_SURFOVERADDR( 9),
+        GR3D_GLOBAL_SURFOVERADDR(10),
+        GR3D_GLOBAL_SURFOVERADDR(11),
+        GR3D_GLOBAL_SURFOVERADDR(12),
+        GR3D_GLOBAL_SURFOVERADDR(13),
+        GR3D_GLOBAL_SURFOVERADDR(14),
+        GR3D_GLOBAL_SURFOVERADDR(15),
+        GR3D_GLOBAL_SAMP01SURFADDR( 0),
+        GR3D_GLOBAL_SAMP01SURFADDR( 1),
+        GR3D_GLOBAL_SAMP01SURFADDR( 2),
+        GR3D_GLOBAL_SAMP01SURFADDR( 3),
+        GR3D_GLOBAL_SAMP01SURFADDR( 4),
+        GR3D_GLOBAL_SAMP01SURFADDR( 5),
+        GR3D_GLOBAL_SAMP01SURFADDR( 6),
+        GR3D_GLOBAL_SAMP01SURFADDR( 7),
+        GR3D_GLOBAL_SAMP01SURFADDR( 8),
+        GR3D_GLOBAL_SAMP01SURFADDR( 9),
+        GR3D_GLOBAL_SAMP01SURFADDR(10),
+        GR3D_GLOBAL_SAMP01SURFADDR(11),
+        GR3D_GLOBAL_SAMP01SURFADDR(12),
+        GR3D_GLOBAL_SAMP01SURFADDR(13),
+        GR3D_GLOBAL_SAMP01SURFADDR(14),
+        GR3D_GLOBAL_SAMP01SURFADDR(15),
+        GR3D_GLOBAL_SAMP23SURFADDR( 0),
+        GR3D_GLOBAL_SAMP23SURFADDR( 1),
+        GR3D_GLOBAL_SAMP23SURFADDR( 2),
+        GR3D_GLOBAL_SAMP23SURFADDR( 3),
+        GR3D_GLOBAL_SAMP23SURFADDR( 4),
+        GR3D_GLOBAL_SAMP23SURFADDR( 5),
+        GR3D_GLOBAL_SAMP23SURFADDR( 6),
+        GR3D_GLOBAL_SAMP23SURFADDR( 7),
+        GR3D_GLOBAL_SAMP23SURFADDR( 8),
+        GR3D_GLOBAL_SAMP23SURFADDR( 9),
+        GR3D_GLOBAL_SAMP23SURFADDR(10),
+        GR3D_GLOBAL_SAMP23SURFADDR(11),
+        GR3D_GLOBAL_SAMP23SURFADDR(12),
+        GR3D_GLOBAL_SAMP23SURFADDR(13),
+        GR3D_GLOBAL_SAMP23SURFADDR(14),
+        GR3D_GLOBAL_SAMP23SURFADDR(15),
+};
+
+static int gr3d_probe(struct platform_device *pdev)
+{
+        struct device_node *np = pdev->dev.of_node;
+        struct host1x_syncpt **syncpts;
+        struct gr3d *gr3d;
+        unsigned int i;
+        int err;
+
+        gr3d = devm_kzalloc(&pdev->dev, sizeof(*gr3d), GFP_KERNEL);
+        if (!gr3d)
+                return -ENOMEM;
+
+        syncpts = devm_kzalloc(&pdev->dev, sizeof(*syncpts), GFP_KERNEL);
+        if (!syncpts)
+                return -ENOMEM;
+
+        gr3d->clk = devm_clk_get(&pdev->dev, NULL);
+        if (IS_ERR(gr3d->clk)) {
+                dev_err(&pdev->dev, "cannot get clock\n");
+                return PTR_ERR(gr3d->clk);
+        }
+
+        if (of_device_is_compatible(np, "nvidia,tegra30-gr3d")) {
+                gr3d->clk_secondary = devm_clk_get(&pdev->dev, "3d2");
+                if (IS_ERR(gr3d->clk)) {
+                        dev_err(&pdev->dev, "cannot get secondary clock\n");
+                        return PTR_ERR(gr3d->clk);
+                }
+        }
+
+        err = tegra_powergate_sequence_power_up(TEGRA_POWERGATE_3D, gr3d->clk);
+        if (err < 0) {
+                dev_err(&pdev->dev, "failed to power up 3D unit\n");
+                return err;
+        }
+
+        if (gr3d->clk_secondary) {
+                err = tegra_powergate_sequence_power_up(TEGRA_POWERGATE_3D1,
+                                                        gr3d->clk_secondary);
+                if (err < 0) {
+                        dev_err(&pdev->dev,
+                                "failed to power up secondary 3D unit\n");
+                        return err;
+                }
+        }
+
+        INIT_LIST_HEAD(&gr3d->client.base.list);
+        gr3d->client.base.ops = &gr3d_client_ops;
+        gr3d->client.base.dev = &pdev->dev;
+        gr3d->client.base.class = HOST1X_CLASS_GR3D;
+        gr3d->client.base.syncpts = syncpts;
+        gr3d->client.base.num_syncpts = 1;
+
+        INIT_LIST_HEAD(&gr3d->client.list);
+        gr3d->client.ops = &gr3d_ops;
+
+        err = host1x_client_register(&gr3d->client.base);
+        if (err < 0) {
+                dev_err(&pdev->dev, "failed to register host1x client: %d\n",
+                        err);
+                return err;
+        }
+
+        /* initialize address register map */
+        for (i = 0; i < ARRAY_SIZE(gr3d_addr_regs); i++)
+                set_bit(gr3d_addr_regs[i], gr3d->addr_regs);
+
+        platform_set_drvdata(pdev, gr3d);
+
+        return 0;
+}
+
+static int gr3d_remove(struct platform_device *pdev)
+{
+        struct gr3d *gr3d = platform_get_drvdata(pdev);
+        int err;
+
+        err = host1x_client_unregister(&gr3d->client.base);
+        if (err < 0) {
+                dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
+                        err);
+                return err;
+        }
+
+        if (gr3d->clk_secondary) {
+                tegra_powergate_power_off(TEGRA_POWERGATE_3D1);
+                clk_disable_unprepare(gr3d->clk_secondary);
+        }
+
+        tegra_powergate_power_off(TEGRA_POWERGATE_3D);
+        clk_disable_unprepare(gr3d->clk);
+
+        return 0;
+}
+
+struct platform_driver tegra_gr3d_driver = {
+        .driver = {
+                .name = "tegra-gr3d",
+                .of_match_table = tegra_gr3d_match,
+        },
+        .probe = gr3d_probe,
+        .remove = gr3d_remove,
+};
diff --git a/drivers/gpu/drm/tegra/gr3d.h b/drivers/gpu/drm/tegra/gr3d.h
new file mode 100644
index 000000000000..0c30a1351c83
--- /dev/null
+++ b/drivers/gpu/drm/tegra/gr3d.h
@@ -0,0 +1,27 @@
+/*
+ * Copyright (C) 2013 NVIDIA Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef TEGRA_GR3D_H
+#define TEGRA_GR3D_H
+
+#define GR3D_IDX_ATTRIBUTE(x)           (0x100 + (x) * 2)
+#define GR3D_IDX_INDEX_BASE             0x121
+#define GR3D_QR_ZTAG_ADDR               0x415
+#define GR3D_QR_CTAG_ADDR               0x417
+#define GR3D_QR_CZ_ADDR                 0x419
+#define GR3D_TEX_TEX_ADDR(x)            (0x710 + (x))
+#define GR3D_DW_MEMORY_OUTPUT_ADDRESS   0x904
+#define GR3D_GLOBAL_SURFADDR(x)         (0xe00 + (x))
+#define GR3D_GLOBAL_SPILLSURFADDR       0xe2a
+#define GR3D_GLOBAL_SURFOVERADDR(x)     (0xe30 + (x))
+#define GR3D_GLOBAL_SAMP01SURFADDR(x)   (0xe50 + (x))
+#define GR3D_GLOBAL_SAMP23SURFADDR(x)   (0xe60 + (x))
+
+#define GR3D_NUM_REGS                   0xe88
+
+#endif
diff --git a/drivers/gpu/host1x/drm/hdmi.c b/drivers/gpu/drm/tegra/hdmi.c
index 644d95c7d489..0cd9bc2056e8 100644
--- a/drivers/gpu/host1x/drm/hdmi.c
+++ b/drivers/gpu/drm/tegra/hdmi.c
@@ -8,21 +8,33 @@
  */
 
 #include <linux/clk.h>
+#include <linux/clk/tegra.h>
 #include <linux/debugfs.h>
-#include <linux/gpio.h>
 #include <linux/hdmi.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
 #include <linux/regulator/consumer.h>
-#include <linux/clk/tegra.h>
-
-#include <drm/drm_edid.h>
 
 #include "hdmi.h"
 #include "drm.h"
 #include "dc.h"
-#include "host1x_client.h"
+
+struct tmds_config {
+        unsigned int pclk;
+        u32 pll0;
+        u32 pll1;
+        u32 pe_current;
+        u32 drive_current;
+        u32 peak_current;
+};
+
+struct tegra_hdmi_config {
+        const struct tmds_config *tmds;
+        unsigned int num_tmds;
+
+        unsigned long fuse_override_offset;
+        unsigned long fuse_override_value;
+
+        bool has_sor_io_peak_current;
+};
 
 struct tegra_hdmi {
         struct host1x_client client;
@@ -38,6 +50,8 @@ struct tegra_hdmi {
         struct clk *clk_parent;
         struct clk *clk;
 
+        const struct tegra_hdmi_config *config;
+
         unsigned int audio_source;
         unsigned int audio_freq;
         bool stereo;
@@ -143,15 +157,7 @@ static const struct tegra_hdmi_audio_config tegra_hdmi_audio_192k[] = {
         {         0,     0,      0,     0 },
 };
 
-struct tmds_config {
-        unsigned int pclk;
-        u32 pll0;
-        u32 pll1;
-        u32 pe_current;
-        u32 drive_current;
-};
-
-static const struct tmds_config tegra2_tmds_config[] = {
+static const struct tmds_config tegra20_tmds_config[] = {
         { /* slow pixel clock modes */
                 .pclk = 27000000,
                 .pll0 = SOR_PLL_BG_V17_S(3) | SOR_PLL_ICHPMP(1) |
@@ -184,7 +190,7 @@ static const struct tmds_config tegra2_tmds_config[] = {
         },
 };
 
-static const struct tmds_config tegra3_tmds_config[] = {
+static const struct tmds_config tegra30_tmds_config[] = {
         { /* 480p modes */
                 .pclk = 27000000,
                 .pll0 = SOR_PLL_BG_V17_S(3) | SOR_PLL_ICHPMP(1) |
@@ -230,6 +236,85 @@ static const struct tmds_config tegra3_tmds_config[] = {
         },
 };
 
+static const struct tmds_config tegra114_tmds_config[] = {
+        { /* 480p/576p / 25.2MHz/27MHz modes */
+                .pclk = 27000000,
+                .pll0 = SOR_PLL_ICHPMP(1) | SOR_PLL_BG_V17_S(3) |
+                        SOR_PLL_VCOCAP(0) | SOR_PLL_RESISTORSEL,
+                .pll1 = SOR_PLL_LOADADJ(3) | SOR_PLL_TMDS_TERMADJ(0),
+                .pe_current = PE_CURRENT0(PE_CURRENT_0_mA_T114) |
+                        PE_CURRENT1(PE_CURRENT_0_mA_T114) |
+                        PE_CURRENT2(PE_CURRENT_0_mA_T114) |
+                        PE_CURRENT3(PE_CURRENT_0_mA_T114),
+                .drive_current =
+                        DRIVE_CURRENT_LANE0_T114(DRIVE_CURRENT_10_400_mA_T114) |
+                        DRIVE_CURRENT_LANE1_T114(DRIVE_CURRENT_10_400_mA_T114) |
+                        DRIVE_CURRENT_LANE2_T114(DRIVE_CURRENT_10_400_mA_T114) |
+                        DRIVE_CURRENT_LANE3_T114(DRIVE_CURRENT_10_400_mA_T114),
+                .peak_current = PEAK_CURRENT_LANE0(PEAK_CURRENT_0_000_mA) |
+                        PEAK_CURRENT_LANE1(PEAK_CURRENT_0_000_mA) |
+                        PEAK_CURRENT_LANE2(PEAK_CURRENT_0_000_mA) |
+                        PEAK_CURRENT_LANE3(PEAK_CURRENT_0_000_mA),
+        }, { /* 720p / 74.25MHz modes */
+                .pclk = 74250000,
+                .pll0 = SOR_PLL_ICHPMP(1) | SOR_PLL_BG_V17_S(3) |
+                        SOR_PLL_VCOCAP(1) | SOR_PLL_RESISTORSEL,
+                .pll1 = SOR_PLL_PE_EN | SOR_PLL_LOADADJ(3) |
+                        SOR_PLL_TMDS_TERMADJ(0),
+                .pe_current = PE_CURRENT0(PE_CURRENT_15_mA_T114) |
+                        PE_CURRENT1(PE_CURRENT_15_mA_T114) |
+                        PE_CURRENT2(PE_CURRENT_15_mA_T114) |
+                        PE_CURRENT3(PE_CURRENT_15_mA_T114),
+                .drive_current =
+                        DRIVE_CURRENT_LANE0_T114(DRIVE_CURRENT_10_400_mA_T114) |
+                        DRIVE_CURRENT_LANE1_T114(DRIVE_CURRENT_10_400_mA_T114) |
+                        DRIVE_CURRENT_LANE2_T114(DRIVE_CURRENT_10_400_mA_T114) |
+                        DRIVE_CURRENT_LANE3_T114(DRIVE_CURRENT_10_400_mA_T114),
+                .peak_current = PEAK_CURRENT_LANE0(PEAK_CURRENT_0_000_mA) |
+                        PEAK_CURRENT_LANE1(PEAK_CURRENT_0_000_mA) |
+                        PEAK_CURRENT_LANE2(PEAK_CURRENT_0_000_mA) |
+                        PEAK_CURRENT_LANE3(PEAK_CURRENT_0_000_mA),
+        }, { /* 1080p / 148.5MHz modes */
+                .pclk = 148500000,
+                .pll0 = SOR_PLL_ICHPMP(1) | SOR_PLL_BG_V17_S(3) |
+                        SOR_PLL_VCOCAP(3) | SOR_PLL_RESISTORSEL,
+                .pll1 = SOR_PLL_PE_EN | SOR_PLL_LOADADJ(3) |
+                        SOR_PLL_TMDS_TERMADJ(0),
+                .pe_current = PE_CURRENT0(PE_CURRENT_10_mA_T114) |
+                        PE_CURRENT1(PE_CURRENT_10_mA_T114) |
+                        PE_CURRENT2(PE_CURRENT_10_mA_T114) |
+                        PE_CURRENT3(PE_CURRENT_10_mA_T114),
+                .drive_current =
+                        DRIVE_CURRENT_LANE0_T114(DRIVE_CURRENT_12_400_mA_T114) |
+                        DRIVE_CURRENT_LANE1_T114(DRIVE_CURRENT_12_400_mA_T114) |
+                        DRIVE_CURRENT_LANE2_T114(DRIVE_CURRENT_12_400_mA_T114) |
+                        DRIVE_CURRENT_LANE3_T114(DRIVE_CURRENT_12_400_mA_T114),
+                .peak_current = PEAK_CURRENT_LANE0(PEAK_CURRENT_0_000_mA) |
+                        PEAK_CURRENT_LANE1(PEAK_CURRENT_0_000_mA) |
+                        PEAK_CURRENT_LANE2(PEAK_CURRENT_0_000_mA) |
+                        PEAK_CURRENT_LANE3(PEAK_CURRENT_0_000_mA),
+        }, { /* 225/297MHz modes */
+                .pclk = UINT_MAX,
+                .pll0 = SOR_PLL_ICHPMP(1) | SOR_PLL_BG_V17_S(3) |
+                        SOR_PLL_VCOCAP(0xf) | SOR_PLL_RESISTORSEL,
+                .pll1 = SOR_PLL_LOADADJ(3) | SOR_PLL_TMDS_TERMADJ(7)
+                        | SOR_PLL_TMDS_TERM_ENABLE,
+                .pe_current = PE_CURRENT0(PE_CURRENT_0_mA_T114) |
+                        PE_CURRENT1(PE_CURRENT_0_mA_T114) |
+                        PE_CURRENT2(PE_CURRENT_0_mA_T114) |
+                        PE_CURRENT3(PE_CURRENT_0_mA_T114),
+                .drive_current =
+                        DRIVE_CURRENT_LANE0_T114(DRIVE_CURRENT_25_200_mA_T114) |
+                        DRIVE_CURRENT_LANE1_T114(DRIVE_CURRENT_25_200_mA_T114) |
+                        DRIVE_CURRENT_LANE2_T114(DRIVE_CURRENT_25_200_mA_T114) |
+                        DRIVE_CURRENT_LANE3_T114(DRIVE_CURRENT_19_200_mA_T114),
+                .peak_current = PEAK_CURRENT_LANE0(PEAK_CURRENT_3_000_mA) |
+                        PEAK_CURRENT_LANE1(PEAK_CURRENT_3_000_mA) |
+                        PEAK_CURRENT_LANE2(PEAK_CURRENT_3_000_mA) |
+                        PEAK_CURRENT_LANE3(PEAK_CURRENT_0_800_mA),
+        },
+};
+
 static const struct tegra_hdmi_audio_config *
 tegra_hdmi_get_audio_config(unsigned int audio_freq, unsigned int pclk)
 {
@@ -511,7 +596,7 @@ static void tegra_hdmi_setup_audio_infoframe(struct tegra_hdmi *hdmi)
 
         err = hdmi_audio_infoframe_init(&frame);
         if (err < 0) {
-                dev_err(hdmi->dev, "failed to initialize audio infoframe: %d\n",
+                dev_err(hdmi->dev, "failed to setup audio infoframe: %zd\n",
                         err);
                 return;
         }
@@ -531,7 +616,7 @@ static void tegra_hdmi_setup_audio_infoframe(struct tegra_hdmi *hdmi)
          * contain 7 bytes. Including the 3 byte header only the first 10
          * bytes can be programmed.
          */
-        tegra_hdmi_write_infopack(hdmi, buffer, min(10, err));
+        tegra_hdmi_write_infopack(hdmi, buffer, min_t(size_t, 10, err));
 
         tegra_hdmi_writel(hdmi, INFOFRAME_CTRL_ENABLE,
                           HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_CTRL);
@@ -577,8 +662,28 @@ static void tegra_hdmi_setup_tmds(struct tegra_hdmi *hdmi,
         tegra_hdmi_writel(hdmi, tmds->pll1, HDMI_NV_PDISP_SOR_PLL1);
         tegra_hdmi_writel(hdmi, tmds->pe_current, HDMI_NV_PDISP_PE_CURRENT);
 
-        value = tmds->drive_current | DRIVE_CURRENT_FUSE_OVERRIDE;
-        tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_SOR_LANE_DRIVE_CURRENT);
+        tegra_hdmi_writel(hdmi, tmds->drive_current,
+                          HDMI_NV_PDISP_SOR_LANE_DRIVE_CURRENT);
+
+        value = tegra_hdmi_readl(hdmi, hdmi->config->fuse_override_offset);
+        value |= hdmi->config->fuse_override_value;
+        tegra_hdmi_writel(hdmi, value, hdmi->config->fuse_override_offset);
+
+        if (hdmi->config->has_sor_io_peak_current)
+                tegra_hdmi_writel(hdmi, tmds->peak_current,
+                                  HDMI_NV_PDISP_SOR_IO_PEAK_CURRENT);
+}
+
+static bool tegra_output_is_hdmi(struct tegra_output *output)
+{
+        struct edid *edid;
+
+        if (!output->connector.edid_blob_ptr)
+                return false;
+
+        edid = (struct edid *)output->connector.edid_blob_ptr->data;
+
+        return drm_detect_hdmi_monitor(edid);
 }
 
 static int tegra_output_hdmi_enable(struct tegra_output *output)
@@ -589,23 +694,17 @@ static int tegra_output_hdmi_enable(struct tegra_output *output)
         struct tegra_hdmi *hdmi = to_hdmi(output);
         struct device_node *node = hdmi->dev->of_node;
         unsigned int pulse_start, div82, pclk;
-        const struct tmds_config *tmds;
-        unsigned int num_tmds;
         unsigned long value;
         int retries = 1000;
         int err;
 
+        hdmi->dvi = !tegra_output_is_hdmi(output);
+
         pclk = mode->clock * 1000;
         h_sync_width = mode->hsync_end - mode->hsync_start;
         h_back_porch = mode->htotal - mode->hsync_end;
         h_front_porch = mode->hsync_start - mode->hdisplay;
 
-        err = regulator_enable(hdmi->vdd);
-        if (err < 0) {
-                dev_err(hdmi->dev, "failed to enable VDD regulator: %d\n", err);
-                return err;
-        }
-
         err = regulator_enable(hdmi->pll);
         if (err < 0) {
                 dev_err(hdmi->dev, "failed to enable PLL regulator: %d\n", err);
@@ -710,17 +809,9 @@ static int tegra_output_hdmi_enable(struct tegra_output *output)
         tegra_hdmi_setup_stereo_infoframe(hdmi);
 
         /* TMDS CONFIG */
-        if (of_device_is_compatible(node, "nvidia,tegra30-hdmi")) {
-                num_tmds = ARRAY_SIZE(tegra3_tmds_config);
-                tmds = tegra3_tmds_config;
-        } else {
-                num_tmds = ARRAY_SIZE(tegra2_tmds_config);
-                tmds = tegra2_tmds_config;
-        }
-
-        for (i = 0; i < num_tmds; i++) {
-                if (pclk <= tmds[i].pclk) {
-                        tegra_hdmi_setup_tmds(hdmi, &tmds[i]);
+        for (i = 0; i < hdmi->config->num_tmds; i++) {
+                if (pclk <= hdmi->config->tmds[i].pclk) {
+                        tegra_hdmi_setup_tmds(hdmi, &hdmi->config->tmds[i]);
                         break;
                 }
         }
@@ -824,7 +915,6 @@ static int tegra_output_hdmi_disable(struct tegra_output *output)
         tegra_periph_reset_assert(hdmi->clk);
         clk_disable(hdmi->clk);
         regulator_disable(hdmi->pll);
-        regulator_disable(hdmi->vdd);
 
         return 0;
 }
@@ -1055,6 +1145,7 @@ static int tegra_hdmi_show_regs(struct seq_file *s, void *data)
         DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_CNTRL0);
         DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_HDA_ELD_BUFWR);
         DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_HDA_PRESENSE);
+        DUMP_REG(HDMI_NV_PDISP_SOR_IO_PEAK_CURRENT);
 
 #undef DUMP_REG
 
@@ -1122,24 +1213,31 @@ static int tegra_hdmi_debugfs_exit(struct tegra_hdmi *hdmi)
         return 0;
 }
 
-static int tegra_hdmi_drm_init(struct host1x_client *client,
-                               struct drm_device *drm)
+static int tegra_hdmi_init(struct host1x_client *client)
 {
+        struct tegra_drm *tegra = dev_get_drvdata(client->parent);
         struct tegra_hdmi *hdmi = host1x_client_to_hdmi(client);
         int err;
 
+        err = regulator_enable(hdmi->vdd);
+        if (err < 0) {
+                dev_err(client->dev, "failed to enable VDD regulator: %d\n",
+                        err);
+                return err;
+        }
+
         hdmi->output.type = TEGRA_OUTPUT_HDMI;
         hdmi->output.dev = client->dev;
         hdmi->output.ops = &hdmi_ops;
 
-        err = tegra_output_init(drm, &hdmi->output);
+        err = tegra_output_init(tegra->drm, &hdmi->output);
         if (err < 0) {
                 dev_err(client->dev, "output setup failed: %d\n", err);
                 return err;
         }
 
         if (IS_ENABLED(CONFIG_DEBUG_FS)) {
-                err = tegra_hdmi_debugfs_init(hdmi, drm->primary);
+                err = tegra_hdmi_debugfs_init(hdmi, tegra->drm->primary);
                 if (err < 0)
                         dev_err(client->dev, "debugfs setup failed: %d\n", err);
         }
@@ -1147,7 +1245,7 @@ static int tegra_hdmi_drm_init(struct host1x_client *client,
         return 0;
 }
 
-static int tegra_hdmi_drm_exit(struct host1x_client *client)
+static int tegra_hdmi_exit(struct host1x_client *client)
 {
         struct tegra_hdmi *hdmi = host1x_client_to_hdmi(client);
         int err;
@@ -1171,25 +1269,63 @@ static int tegra_hdmi_drm_exit(struct host1x_client *client)
                 return err;
         }
 
+        regulator_disable(hdmi->vdd);
+
         return 0;
 }
 
 static const struct host1x_client_ops hdmi_client_ops = {
-        .drm_init = tegra_hdmi_drm_init,
-        .drm_exit = tegra_hdmi_drm_exit,
+        .init = tegra_hdmi_init,
+        .exit = tegra_hdmi_exit,
+};
+
+static const struct tegra_hdmi_config tegra20_hdmi_config = {
+        .tmds = tegra20_tmds_config,
+        .num_tmds = ARRAY_SIZE(tegra20_tmds_config),
+        .fuse_override_offset = HDMI_NV_PDISP_SOR_LANE_DRIVE_CURRENT,
+        .fuse_override_value = 1 << 31,
+        .has_sor_io_peak_current = false,
+};
+
+static const struct tegra_hdmi_config tegra30_hdmi_config = {
+        .tmds = tegra30_tmds_config,
+        .num_tmds = ARRAY_SIZE(tegra30_tmds_config),
+        .fuse_override_offset = HDMI_NV_PDISP_SOR_LANE_DRIVE_CURRENT,
+        .fuse_override_value = 1 << 31,
+        .has_sor_io_peak_current = false,
+};
+
+static const struct tegra_hdmi_config tegra114_hdmi_config = {
+        .tmds = tegra114_tmds_config,
+        .num_tmds = ARRAY_SIZE(tegra114_tmds_config),
+        .fuse_override_offset = HDMI_NV_PDISP_SOR_PAD_CTLS0,
+        .fuse_override_value = 1 << 31,
+        .has_sor_io_peak_current = true,
+};
+
+static const struct of_device_id tegra_hdmi_of_match[] = {
+        { .compatible = "nvidia,tegra114-hdmi", .data = &tegra114_hdmi_config },
+        { .compatible = "nvidia,tegra30-hdmi", .data = &tegra30_hdmi_config },
+        { .compatible = "nvidia,tegra20-hdmi", .data = &tegra20_hdmi_config },
+        { },
 };
 
 static int tegra_hdmi_probe(struct platform_device *pdev)
 {
-        struct host1x_drm *host1x = host1x_get_drm_data(pdev->dev.parent);
+        const struct of_device_id *match;
         struct tegra_hdmi *hdmi;
         struct resource *regs;
         int err;
 
+        match = of_match_node(tegra_hdmi_of_match, pdev->dev.of_node);
+        if (!match)
+                return -ENODEV;
+
         hdmi = devm_kzalloc(&pdev->dev, sizeof(*hdmi), GFP_KERNEL);
         if (!hdmi)
                 return -ENOMEM;
 
+        hdmi->config = match->data;
         hdmi->dev = &pdev->dev;
         hdmi->audio_source = AUTO;
         hdmi->audio_freq = 44100;
@@ -1234,7 +1370,7 @@ static int tegra_hdmi_probe(struct platform_device *pdev)
 
         hdmi->output.dev = &pdev->dev;
 
-        err = tegra_output_parse_dt(&hdmi->output);
+        err = tegra_output_probe(&hdmi->output);
         if (err < 0)
                 return err;
 
@@ -1252,11 +1388,11 @@ static int tegra_hdmi_probe(struct platform_device *pdev)
 
         hdmi->irq = err;
 
-        hdmi->client.ops = &hdmi_client_ops;
         INIT_LIST_HEAD(&hdmi->client.list);
+        hdmi->client.ops = &hdmi_client_ops;
         hdmi->client.dev = &pdev->dev;
 
-        err = host1x_register_client(host1x, &hdmi->client);
+        err = host1x_client_register(&hdmi->client);
         if (err < 0) {
                 dev_err(&pdev->dev, "failed to register host1x client: %d\n",
                         err);
@@ -1270,29 +1406,28 @@ static int tegra_hdmi_probe(struct platform_device *pdev)
 
 static int tegra_hdmi_remove(struct platform_device *pdev)
 {
-        struct host1x_drm *host1x = host1x_get_drm_data(pdev->dev.parent);
         struct tegra_hdmi *hdmi = platform_get_drvdata(pdev);
         int err;
 
-        err = host1x_unregister_client(host1x, &hdmi->client);
+        err = host1x_client_unregister(&hdmi->client);
         if (err < 0) {
                 dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
                         err);
                 return err;
         }
 
+        err = tegra_output_remove(&hdmi->output);
+        if (err < 0) {
+                dev_err(&pdev->dev, "failed to remove output: %d\n", err);
+                return err;
+        }
+
         clk_unprepare(hdmi->clk_parent);
         clk_unprepare(hdmi->clk);
 
         return 0;
 }
 
-static struct of_device_id tegra_hdmi_of_match[] = {
-        { .compatible = "nvidia,tegra30-hdmi", },
-        { .compatible = "nvidia,tegra20-hdmi", },
-        { },
-};
-
 struct platform_driver tegra_hdmi_driver = {
         .driver = {
                 .name = "tegra-hdmi",
diff --git a/drivers/gpu/host1x/drm/hdmi.h b/drivers/gpu/drm/tegra/hdmi.h
index 52ac36e08ccb..0aebc485f7fa 100644
--- a/drivers/gpu/host1x/drm/hdmi.h
+++ b/drivers/gpu/drm/tegra/hdmi.h
@@ -233,7 +233,10 @@
 #define DRIVE_CURRENT_LANE1(x)      (((x) & 0x3f) <<  8)
 #define DRIVE_CURRENT_LANE2(x)      (((x) & 0x3f) << 16)
 #define DRIVE_CURRENT_LANE3(x)      (((x) & 0x3f) << 24)
-#define DRIVE_CURRENT_FUSE_OVERRIDE (1 << 31)
+#define DRIVE_CURRENT_LANE0_T114(x) (((x) & 0x7f) <<  0)
+#define DRIVE_CURRENT_LANE1_T114(x) (((x) & 0x7f) <<  8)
+#define DRIVE_CURRENT_LANE2_T114(x) (((x) & 0x7f) << 16)
+#define DRIVE_CURRENT_LANE3_T114(x) (((x) & 0x7f) << 24)
 
 #define DRIVE_CURRENT_1_500_mA  0x00
 #define DRIVE_CURRENT_1_875_mA  0x01
@@ -299,6 +302,79 @@
 #define DRIVE_CURRENT_24_375_mA 0x3d
 #define DRIVE_CURRENT_24_750_mA 0x3e
 
+#define DRIVE_CURRENT_0_000_mA_T114 0x00
+#define DRIVE_CURRENT_0_400_mA_T114 0x01
+#define DRIVE_CURRENT_0_800_mA_T114 0x02
+#define DRIVE_CURRENT_1_200_mA_T114 0x03
+#define DRIVE_CURRENT_1_600_mA_T114 0x04
+#define DRIVE_CURRENT_2_000_mA_T114 0x05
+#define DRIVE_CURRENT_2_400_mA_T114 0x06
+#define DRIVE_CURRENT_2_800_mA_T114 0x07
+#define DRIVE_CURRENT_3_200_mA_T114 0x08
+#define DRIVE_CURRENT_3_600_mA_T114 0x09
+#define DRIVE_CURRENT_4_000_mA_T114 0x0a
+#define DRIVE_CURRENT_4_400_mA_T114 0x0b
+#define DRIVE_CURRENT_4_800_mA_T114 0x0c
+#define DRIVE_CURRENT_5_200_mA_T114 0x0d
+#define DRIVE_CURRENT_5_600_mA_T114 0x0e
+#define DRIVE_CURRENT_6_000_mA_T114 0x0f
+#define DRIVE_CURRENT_6_400_mA_T114 0x10
+#define DRIVE_CURRENT_6_800_mA_T114 0x11
+#define DRIVE_CURRENT_7_200_mA_T114 0x12
+#define DRIVE_CURRENT_7_600_mA_T114 0x13
+#define DRIVE_CURRENT_8_000_mA_T114 0x14
+#define DRIVE_CURRENT_8_400_mA_T114 0x15
+#define DRIVE_CURRENT_8_800_mA_T114 0x16
+#define DRIVE_CURRENT_9_200_mA_T114 0x17
+#define DRIVE_CURRENT_9_600_mA_T114 0x18
+#define DRIVE_CURRENT_10_000_mA_T114 0x19
+#define DRIVE_CURRENT_10_400_mA_T114 0x1a
+#define DRIVE_CURRENT_10_800_mA_T114 0x1b
+#define DRIVE_CURRENT_11_200_mA_T114 0x1c
+#define DRIVE_CURRENT_11_600_mA_T114 0x1d
+#define DRIVE_CURRENT_12_000_mA_T114 0x1e
+#define DRIVE_CURRENT_12_400_mA_T114 0x1f
+#define DRIVE_CURRENT_12_800_mA_T114 0x20
+#define DRIVE_CURRENT_13_200_mA_T114 0x21
+#define DRIVE_CURRENT_13_600_mA_T114 0x22
+#define DRIVE_CURRENT_14_000_mA_T114 0x23
+#define DRIVE_CURRENT_14_400_mA_T114 0x24
+#define DRIVE_CURRENT_14_800_mA_T114 0x25
+#define DRIVE_CURRENT_15_200_mA_T114 0x26
+#define DRIVE_CURRENT_15_600_mA_T114 0x27
+#define DRIVE_CURRENT_16_000_mA_T114 0x28
+#define DRIVE_CURRENT_16_400_mA_T114 0x29
+#define DRIVE_CURRENT_16_800_mA_T114 0x2a
+#define DRIVE_CURRENT_17_200_mA_T114 0x2b
+#define DRIVE_CURRENT_17_600_mA_T114 0x2c
+#define DRIVE_CURRENT_18_000_mA_T114 0x2d
+#define DRIVE_CURRENT_18_400_mA_T114 0x2e
+#define DRIVE_CURRENT_18_800_mA_T114 0x2f
+#define DRIVE_CURRENT_19_200_mA_T114 0x30
+#define DRIVE_CURRENT_19_600_mA_T114 0x31
+#define DRIVE_CURRENT_20_000_mA_T114 0x32
+#define DRIVE_CURRENT_20_400_mA_T114 0x33
+#define DRIVE_CURRENT_20_800_mA_T114 0x34
+#define DRIVE_CURRENT_21_200_mA_T114 0x35
+#define DRIVE_CURRENT_21_600_mA_T114 0x36
+#define DRIVE_CURRENT_22_000_mA_T114 0x37
+#define DRIVE_CURRENT_22_400_mA_T114 0x38
+#define DRIVE_CURRENT_22_800_mA_T114 0x39
+#define DRIVE_CURRENT_23_200_mA_T114 0x3a
+#define DRIVE_CURRENT_23_600_mA_T114 0x3b
+#define DRIVE_CURRENT_24_000_mA_T114 0x3c
+#define DRIVE_CURRENT_24_400_mA_T114 0x3d
+#define DRIVE_CURRENT_24_800_mA_T114 0x3e
+#define DRIVE_CURRENT_25_200_mA_T114 0x3f
+#define DRIVE_CURRENT_25_400_mA_T114 0x40
+#define DRIVE_CURRENT_25_800_mA_T114 0x41
+#define DRIVE_CURRENT_26_200_mA_T114 0x42
+#define DRIVE_CURRENT_26_600_mA_T114 0x43
+#define DRIVE_CURRENT_27_000_mA_T114 0x44
+#define DRIVE_CURRENT_27_400_mA_T114 0x45
+#define DRIVE_CURRENT_27_800_mA_T114 0x46
+#define DRIVE_CURRENT_28_200_mA_T114 0x47
+
 #define HDMI_NV_PDISP_AUDIO_DEBUG0                              0x7f
 #define HDMI_NV_PDISP_AUDIO_DEBUG1                              0x80
 #define HDMI_NV_PDISP_AUDIO_DEBUG2                              0x81
@@ -358,6 +434,23 @@
 #define PE_CURRENT_7_0_mA 0xe
 #define PE_CURRENT_7_5_mA 0xf
 
+#define PE_CURRENT_0_mA_T114 0x0
+#define PE_CURRENT_1_mA_T114 0x1
+#define PE_CURRENT_2_mA_T114 0x2
+#define PE_CURRENT_3_mA_T114 0x3
+#define PE_CURRENT_4_mA_T114 0x4
+#define PE_CURRENT_5_mA_T114 0x5
+#define PE_CURRENT_6_mA_T114 0x6
+#define PE_CURRENT_7_mA_T114 0x7
+#define PE_CURRENT_8_mA_T114 0x8
+#define PE_CURRENT_9_mA_T114 0x9
+#define PE_CURRENT_10_mA_T114 0xa
+#define PE_CURRENT_11_mA_T114 0xb
+#define PE_CURRENT_12_mA_T114 0xc
+#define PE_CURRENT_13_mA_T114 0xd
+#define PE_CURRENT_14_mA_T114 0xe
+#define PE_CURRENT_15_mA_T114 0xf
+
 #define HDMI_NV_PDISP_KEY_CTRL                                  0x9a
 #define HDMI_NV_PDISP_KEY_DEBUG0                                0x9b
 #define HDMI_NV_PDISP_KEY_DEBUG1                                0x9c
@@ -383,4 +476,61 @@
 #define HDMI_NV_PDISP_SOR_AUDIO_AVAL_1920    0xc5
 #define HDMI_NV_PDISP_SOR_AUDIO_AVAL_DEFAULT 0xc5
 
+#define HDMI_NV_PDISP_SOR_IO_PEAK_CURRENT               0xd1
+#define PEAK_CURRENT_LANE0(x) (((x) & 0x7f) <<  0)
+#define PEAK_CURRENT_LANE1(x) (((x) & 0x7f) <<  8)
+#define PEAK_CURRENT_LANE2(x) (((x) & 0x7f) << 16)
+#define PEAK_CURRENT_LANE3(x) (((x) & 0x7f) << 24)
+
+#define PEAK_CURRENT_0_000_mA 0x00
+#define PEAK_CURRENT_0_200_mA 0x01
+#define PEAK_CURRENT_0_400_mA 0x02
+#define PEAK_CURRENT_0_600_mA 0x03
+#define PEAK_CURRENT_0_800_mA 0x04
+#define PEAK_CURRENT_1_000_mA 0x05
+#define PEAK_CURRENT_1_200_mA 0x06
+#define PEAK_CURRENT_1_400_mA 0x07
+#define PEAK_CURRENT_1_600_mA 0x08
+#define PEAK_CURRENT_1_800_mA 0x09
+#define PEAK_CURRENT_2_000_mA 0x0a
+#define PEAK_CURRENT_2_200_mA 0x0b
+#define PEAK_CURRENT_2_400_mA 0x0c
+#define PEAK_CURRENT_2_600_mA 0x0d
+#define PEAK_CURRENT_2_800_mA 0x0e
+#define PEAK_CURRENT_3_000_mA 0x0f
+#define PEAK_CURRENT_3_200_mA 0x10
+#define PEAK_CURRENT_3_400_mA 0x11
+#define PEAK_CURRENT_3_600_mA 0x12
+#define PEAK_CURRENT_3_800_mA 0x13
+#define PEAK_CURRENT_4_000_mA 0x14
+#define PEAK_CURRENT_4_200_mA 0x15
+#define PEAK_CURRENT_4_400_mA 0x16
+#define PEAK_CURRENT_4_600_mA 0x17
+#define PEAK_CURRENT_4_800_mA 0x18
+#define PEAK_CURRENT_5_000_mA 0x19
+#define PEAK_CURRENT_5_200_mA 0x1a
+#define PEAK_CURRENT_5_400_mA 0x1b
+#define PEAK_CURRENT_5_600_mA 0x1c
+#define PEAK_CURRENT_5_800_mA 0x1d
+#define PEAK_CURRENT_6_000_mA 0x1e
+#define PEAK_CURRENT_6_200_mA 0x1f
+#define PEAK_CURRENT_6_400_mA 0x20
+#define PEAK_CURRENT_6_600_mA 0x21
+#define PEAK_CURRENT_6_800_mA 0x22
+#define PEAK_CURRENT_7_000_mA 0x23
+#define PEAK_CURRENT_7_200_mA 0x24
+#define PEAK_CURRENT_7_400_mA 0x25
+#define PEAK_CURRENT_7_600_mA 0x26
+#define PEAK_CURRENT_7_800_mA 0x27
+#define PEAK_CURRENT_8_000_mA 0x28
+#define PEAK_CURRENT_8_200_mA 0x29
+#define PEAK_CURRENT_8_400_mA 0x2a
+#define PEAK_CURRENT_8_600_mA 0x2b
+#define PEAK_CURRENT_8_800_mA 0x2c
+#define PEAK_CURRENT_9_000_mA 0x2d
+#define PEAK_CURRENT_9_200_mA 0x2e
+#define PEAK_CURRENT_9_400_mA 0x2f
+
+#define HDMI_NV_PDISP_SOR_PAD_CTLS0             0xd2
+
 #endif /* TEGRA_HDMI_H */
diff --git a/drivers/gpu/host1x/drm/output.c b/drivers/gpu/drm/tegra/output.c
index 137ae81ab80e..2cb0065e0578 100644
--- a/drivers/gpu/host1x/drm/output.c
+++ b/drivers/gpu/drm/tegra/output.c
@@ -7,9 +7,7 @@
  * published by the Free Software Foundation.
  */
 
-#include <linux/module.h>
 #include <linux/of_gpio.h>
-#include <linux/i2c.h>
 
 #include "drm.h"
 
@@ -81,10 +79,16 @@ tegra_connector_detect(struct drm_connector *connector, bool force)
         return status;
 }
 
+static void drm_connector_clear(struct drm_connector *connector)
+{
+        memset(connector, 0, sizeof(*connector));
+}
+
 static void tegra_connector_destroy(struct drm_connector *connector)
 {
         drm_sysfs_connector_remove(connector);
         drm_connector_cleanup(connector);
+        drm_connector_clear(connector);
 }
 
 static const struct drm_connector_funcs connector_funcs = {
@@ -94,9 +98,15 @@ static const struct drm_connector_funcs connector_funcs = {
         .destroy = tegra_connector_destroy,
 };
 
+static void drm_encoder_clear(struct drm_encoder *encoder)
+{
+        memset(encoder, 0, sizeof(*encoder));
+}
+
 static void tegra_encoder_destroy(struct drm_encoder *encoder)
 {
         drm_encoder_cleanup(encoder);
+        drm_encoder_clear(encoder);
 }
 
 static const struct drm_encoder_funcs encoder_funcs = {
@@ -151,7 +161,7 @@ static irqreturn_t hpd_irq(int irq, void *data)
         return IRQ_HANDLED;
 }
 
-int tegra_output_parse_dt(struct tegra_output *output)
+int tegra_output_probe(struct tegra_output *output)
 {
         enum of_gpio_flags flags;
         struct device_node *ddc;
@@ -181,14 +191,6 @@ int tegra_output_parse_dt(struct tegra_output *output)
         output->hpd_gpio = of_get_named_gpio_flags(output->of_node,
                                                    "nvidia,hpd-gpio", 0,
                                                    &flags);
-
-        return 0;
-}
-
-int tegra_output_init(struct drm_device *drm, struct tegra_output *output)
-{
-        int connector, encoder, err;
-
         if (gpio_is_valid(output->hpd_gpio)) {
                 unsigned long flags;
 
@@ -202,7 +204,8 @@ int tegra_output_init(struct drm_device *drm, struct tegra_output *output)
                 err = gpio_to_irq(output->hpd_gpio);
                 if (err < 0) {
                         dev_err(output->dev, "gpio_to_irq(): %d\n", err);
-                        goto free_hpd;
+                        gpio_free(output->hpd_gpio);
+                        return err;
                 }
 
                 output->hpd_irq = err;
@@ -215,12 +218,33 @@ int tegra_output_init(struct drm_device *drm, struct tegra_output *output)
                 if (err < 0) {
                         dev_err(output->dev, "failed to request IRQ#%u: %d\n",
                                 output->hpd_irq, err);
-                        goto free_hpd;
+                        gpio_free(output->hpd_gpio);
+                        return err;
                 }
 
                 output->connector.polled = DRM_CONNECTOR_POLL_HPD;
         }
 
+        return 0;
+}
+
+int tegra_output_remove(struct tegra_output *output)
+{
+        if (gpio_is_valid(output->hpd_gpio)) {
+                free_irq(output->hpd_irq, output);
+                gpio_free(output->hpd_gpio);
+        }
+
+        if (output->ddc)
+                put_device(&output->ddc->dev);
+
+        return 0;
+}
+
+int tegra_output_init(struct drm_device *drm, struct tegra_output *output)
+{
+        int connector, encoder;
+
         switch (output->type) {
         case TEGRA_OUTPUT_RGB:
                 connector = DRM_MODE_CONNECTOR_LVDS;
@@ -241,6 +265,7 @@ int tegra_output_init(struct drm_device *drm, struct tegra_output *output)
         drm_connector_init(drm, &output->connector, &connector_funcs,
                            connector);
         drm_connector_helper_add(&output->connector, &connector_helper_funcs);
+        output->connector.dpms = DRM_MODE_DPMS_OFF;
 
         drm_encoder_init(drm, &output->encoder, &encoder_funcs, encoder);
         drm_encoder_helper_add(&output->encoder, &encoder_helper_funcs);
@@ -251,22 +276,9 @@ int tegra_output_init(struct drm_device *drm, struct tegra_output *output)
         output->encoder.possible_crtcs = 0x3;
 
         return 0;
-
-free_hpd:
-        gpio_free(output->hpd_gpio);
-
-        return err;
 }
 
 int tegra_output_exit(struct tegra_output *output)
 {
-        if (gpio_is_valid(output->hpd_gpio)) {
-                free_irq(output->hpd_irq, output);
-                gpio_free(output->hpd_gpio);
-        }
-
-        if (output->ddc)
-                put_device(&output->ddc->dev);
-
         return 0;
 }
diff --git a/drivers/gpu/host1x/drm/rgb.c b/drivers/gpu/drm/tegra/rgb.c
index 5aa66ef7a946..ba47ca4fb880 100644
--- a/drivers/gpu/host1x/drm/rgb.c
+++ b/drivers/gpu/drm/tegra/rgb.c
@@ -8,9 +8,6 @@
  */
 
 #include <linux/clk.h>
-#include <linux/module.h>
-#include <linux/of.h>
-#include <linux/platform_device.h>
 
 #include "drm.h"
 #include "dc.h"
@@ -150,7 +147,7 @@ int tegra_dc_rgb_probe(struct tegra_dc *dc)
         rgb->output.dev = dc->dev;
         rgb->output.of_node = np;
 
-        err = tegra_output_parse_dt(&rgb->output);
+        err = tegra_output_probe(&rgb->output);
         if (err < 0)
                 return err;
 
@@ -177,6 +174,20 @@ int tegra_dc_rgb_probe(struct tegra_dc *dc)
         return 0;
 }
 
+int tegra_dc_rgb_remove(struct tegra_dc *dc)
+{
+        int err;
+
+        if (!dc->rgb)
+                return 0;
+
+        err = tegra_output_remove(dc->rgb);
+        if (err < 0)
+                return err;
+
+        return 0;
+}
+
 int tegra_dc_rgb_init(struct drm_device *drm, struct tegra_dc *dc)
 {
         struct tegra_rgb *rgb = to_rgb(dc->rgb);
diff --git a/drivers/gpu/drm/ttm/Makefile b/drivers/gpu/drm/ttm/Makefile
index b2b33dde2afb..b433b9f040c9 100644
--- a/drivers/gpu/drm/ttm/Makefile
+++ b/drivers/gpu/drm/ttm/Makefile
@@ -5,10 +5,6 @@ ccflags-y := -Iinclude/drm
 ttm-y := ttm_agp_backend.o ttm_memory.o ttm_tt.o ttm_bo.o \
         ttm_bo_util.o ttm_bo_vm.o ttm_module.o \
         ttm_object.o ttm_lock.o ttm_execbuf_util.o ttm_page_alloc.o \
-        ttm_bo_manager.o
-
-ifeq ($(CONFIG_SWIOTLB),y)
-ttm-y += ttm_page_alloc_dma.o
-endif
+        ttm_bo_manager.o ttm_page_alloc_dma.o
 
 obj-$(CONFIG_DRM_TTM) += ttm.o
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index f1a857ec1021..8d5a646ebe6a 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -429,8 +429,20 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
                 sync_obj = driver->sync_obj_ref(bo->sync_obj);
         spin_unlock(&bdev->fence_lock);
 
-        if (!ret)
+        if (!ret) {
+
+                /*
+                 * Make NO_EVICT bos immediately available to
+                 * shrinkers, now that they are queued for
+                 * destruction.
+                 */
+                if (bo->mem.placement & TTM_PL_FLAG_NO_EVICT) {
+                        bo->mem.placement &= ~TTM_PL_FLAG_NO_EVICT;
+                        ttm_bo_add_to_lru(bo);
+                }
+
                 ww_mutex_unlock(&bo->resv->lock);
+        }
 
         kref_get(&bo->list_kref);
         list_add_tail(&bo->ddestroy, &bdev->ddestroy);
@@ -986,24 +998,32 @@ out_unlock:
         return ret;
 }
 
-static int ttm_bo_mem_compat(struct ttm_placement *placement,
-                             struct ttm_mem_reg *mem)
+static bool ttm_bo_mem_compat(struct ttm_placement *placement,
+                              struct ttm_mem_reg *mem,
+                              uint32_t *new_flags)
 {
         int i;
 
         if (mem->mm_node && placement->lpfn != 0 &&
             (mem->start < placement->fpfn ||
              mem->start + mem->num_pages > placement->lpfn))
-                return -1;
+                return false;
 
         for (i = 0; i < placement->num_placement; i++) {
-                if ((placement->placement[i] & mem->placement &
-                        TTM_PL_MASK_CACHING) &&
-                        (placement->placement[i] & mem->placement &
-                        TTM_PL_MASK_MEM))
-                        return i;
+                *new_flags = placement->placement[i];
+                if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
+                    (*new_flags & mem->placement & TTM_PL_MASK_MEM))
+                        return true;
         }
-        return -1;
+
+        for (i = 0; i < placement->num_busy_placement; i++) {
+                *new_flags = placement->busy_placement[i];
+                if ((*new_flags & mem->placement & TTM_PL_MASK_CACHING) &&
+                    (*new_flags & mem->placement & TTM_PL_MASK_MEM))
+                        return true;
+        }
+
+        return false;
 }
 
 int ttm_bo_validate(struct ttm_buffer_object *bo,
@@ -1012,6 +1032,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo,
                         bool no_wait_gpu)
 {
         int ret;
+        uint32_t new_flags;
 
         lockdep_assert_held(&bo->resv->lock.base);
         /* Check that range is valid */
@@ -1022,8 +1043,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo,
         /*
          * Check whether we need to move buffer.
          */
-        ret = ttm_bo_mem_compat(placement, &bo->mem);
-        if (ret < 0) {
+        if (!ttm_bo_mem_compat(placement, &bo->mem, &new_flags)) {
                 ret = ttm_bo_move_buffer(bo, placement, interruptible,
                                          no_wait_gpu);
                 if (ret)
@@ -1033,7 +1053,7 @@ int ttm_bo_validate(struct ttm_buffer_object *bo,
                  * Use the access and other non-mapping-related flag bits from
                  * the compatible memory placement flags to the active flags
                  */
-                ttm_flag_masked(&bo->mem.placement, placement->placement[ret],
+                ttm_flag_masked(&bo->mem.placement, new_flags,
                                 ~TTM_PL_MASK_MEMTYPE);
         }
         /*
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index 7cc904d3a4d1..4834c463c38b 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -343,19 +343,25 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
         if (ret)
                 goto out;
 
+        /*
+         * Single TTM move. NOP.
+         */
         if (old_iomap == NULL && new_iomap == NULL)
                 goto out2;
+
+        /*
+         * Move nonexistent data. NOP.
+         */
         if (old_iomap == NULL && ttm == NULL)
                 goto out2;
 
-        if (ttm->state == tt_unpopulated) {
+        /*
+         * TTM might be null for moves within the same region.
+         */
+        if (ttm && ttm->state == tt_unpopulated) {
                 ret = ttm->bdev->driver->ttm_tt_populate(ttm);
-                if (ret) {
-                        /* if we fail here don't nuke the mm node
-                         * as the bo still owns it */
-                        old_copy.mm_node = NULL;
+                if (ret)
                         goto out1;
-                }
         }
 
         add = 0;
@@ -381,11 +387,8 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
                                                    prot);
                 } else
                         ret = ttm_copy_io_page(new_iomap, old_iomap, page);
-                if (ret) {
-                        /* failing here, means keep old copy as-is */
-                        old_copy.mm_node = NULL;
+                if (ret)
                         goto out1;
-                }
         }
         mb();
 out2:
@@ -403,7 +406,12 @@ out1:
         ttm_mem_reg_iounmap(bdev, old_mem, new_iomap);
 out:
         ttm_mem_reg_iounmap(bdev, &old_copy, old_iomap);
-        ttm_bo_mem_put(bo, &old_copy);
+
+        /*
+         * On error, keep the mm node!
+         */
+        if (!ret)
+                ttm_bo_mem_put(bo, &old_copy);
         return ret;
 }
 EXPORT_SYMBOL(ttm_bo_move_memcpy);
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index 1006c15445e9..ac617f3ecd0c 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -41,6 +41,51 @@
 
 #define TTM_BO_VM_NUM_PREFAULT 16
 
+static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo,
+                                struct vm_area_struct *vma,
+                                struct vm_fault *vmf)
+{
+        struct ttm_bo_device *bdev = bo->bdev;
+        int ret = 0;
+
+        spin_lock(&bdev->fence_lock);
+        if (likely(!test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)))
+                goto out_unlock;
+
+        /*
+         * Quick non-stalling check for idle.
+         */
+        ret = ttm_bo_wait(bo, false, false, true);
+        if (likely(ret == 0))
+                goto out_unlock;
+
+        /*
+         * If possible, avoid waiting for GPU with mmap_sem
+         * held.
+         */
+        if (vmf->flags & FAULT_FLAG_ALLOW_RETRY) {
+                ret = VM_FAULT_RETRY;
+                if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT)
+                        goto out_unlock;
+
+                up_read(&vma->vm_mm->mmap_sem);
+                (void) ttm_bo_wait(bo, false, true, false);
+                goto out_unlock;
+        }
+
+        /*
+         * Ordinary wait.
+         */
+        ret = ttm_bo_wait(bo, false, true, false);
+        if (unlikely(ret != 0))
+                ret = (ret != -ERESTARTSYS) ? VM_FAULT_SIGBUS :
+                        VM_FAULT_NOPAGE;
+
+out_unlock:
+        spin_unlock(&bdev->fence_lock);
+        return ret;
+}
+
 static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
         struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
@@ -57,6 +102,7 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
         int retval = VM_FAULT_NOPAGE;
         struct ttm_mem_type_manager *man =
                 &bdev->man[bo->mem.mem_type];
+        struct vm_area_struct cvma;
 
         /*
          * Work around locking order reversal in fault / nopfn
@@ -91,18 +137,11 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
          * Wait for buffer data in transit, due to a pipelined
          * move.
          */
-
-        spin_lock(&bdev->fence_lock);
-        if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) {
-                ret = ttm_bo_wait(bo, false, true, false);
-                spin_unlock(&bdev->fence_lock);
-                if (unlikely(ret != 0)) {
-                        retval = (ret != -ERESTARTSYS) ?
-                            VM_FAULT_SIGBUS : VM_FAULT_NOPAGE;
-                        goto out_unlock;
-                }
-        } else
-                spin_unlock(&bdev->fence_lock);
+        ret = ttm_bo_vm_fault_idle(bo, vma, vmf);
+        if (unlikely(ret != 0)) {
+                retval = ret;
+                goto out_unlock;
+        }
 
         ret = ttm_mem_io_lock(man, true);
         if (unlikely(ret != 0)) {
@@ -126,26 +165,21 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
         }
 
         /*
-         * Strictly, we're not allowed to modify vma->vm_page_prot here,
-         * since the mmap_sem is only held in read mode. However, we
-         * modify only the caching bits of vma->vm_page_prot and
-         * consider those bits protected by
-         * the bo->mutex, as we should be the only writers.
-         * There shouldn't really be any readers of these bits except
-         * within vm_insert_mixed()? fork?
-         *
-         * TODO: Add a list of vmas to the bo, and change the
-         * vma->vm_page_prot when the object changes caching policy, with
-         * the correct locks held.
+         * Make a local vma copy to modify the page_prot member
+         * and vm_flags if necessary. The vma parameter is protected
+         * by mmap_sem in write mode.
          */
+        cvma = *vma;
+        cvma.vm_page_prot = vm_get_page_prot(cvma.vm_flags);
+
         if (bo->mem.bus.is_iomem) {
-                vma->vm_page_prot = ttm_io_prot(bo->mem.placement,
-                                                vma->vm_page_prot);
+                cvma.vm_page_prot = ttm_io_prot(bo->mem.placement,
+                                                cvma.vm_page_prot);
         } else {
                 ttm = bo->ttm;
-                vma->vm_page_prot = (bo->mem.placement & TTM_PL_FLAG_CACHED) ?
-                    vm_get_page_prot(vma->vm_flags) :
-                    ttm_io_prot(bo->mem.placement, vma->vm_page_prot);
+                if (!(bo->mem.placement & TTM_PL_FLAG_CACHED))
+                        cvma.vm_page_prot = ttm_io_prot(bo->mem.placement,
+                                                        cvma.vm_page_prot);
 
                 /* Allocate all page at once, most common usage */
                 if (ttm->bdev->driver->ttm_tt_populate(ttm)) {
@@ -172,7 +206,7 @@ static int ttm_bo_vm_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
                         pfn = page_to_pfn(page);
                 }
 
-                ret = vm_insert_mixed(vma, address, pfn);
+                ret = vm_insert_mixed(&cvma, address, pfn);
                 /*
                  * Somebody beat us to this PTE or prefaulting to
                  * an already populated PTE, or prefaulting error.
diff --git a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
index 7957beeeaf73..fb8259f69839 100644
--- a/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
+++ b/drivers/gpu/drm/ttm/ttm_page_alloc_dma.c
@@ -33,6 +33,7 @@
  *   when freed).
  */
 
+#if defined(CONFIG_SWIOTLB) || defined(CONFIG_INTEL_IOMMU)
 #define pr_fmt(fmt) "[TTM] " fmt
 
 #include <linux/dma-mapping.h>
@@ -1142,3 +1143,5 @@ int ttm_dma_page_alloc_debugfs(struct seq_file *m, void *data)
         return 0;
 }
 EXPORT_SYMBOL_GPL(ttm_dma_page_alloc_debugfs);
+
+#endif
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
index 96dc84dc34d0..7776e6f0aef6 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_buffer.c
@@ -141,37 +141,374 @@ struct ttm_placement vmw_srf_placement = {
 };
 
 struct vmw_ttm_tt {
-        struct ttm_tt ttm;
+        struct ttm_dma_tt dma_ttm;
         struct vmw_private *dev_priv;
         int gmr_id;
+        struct sg_table sgt;
+        struct vmw_sg_table vsgt;
+        uint64_t sg_alloc_size;
+        bool mapped;
 };
 
+/**
+ * Helper functions to advance a struct vmw_piter iterator.
+ *
+ * @viter: Pointer to the iterator.
+ *
+ * These functions return false if past the end of the list,
+ * true otherwise. Functions are selected depending on the current
+ * DMA mapping mode.
+ */
+static bool __vmw_piter_non_sg_next(struct vmw_piter *viter)
+{
+        return ++(viter->i) < viter->num_pages;
+}
+
+static bool __vmw_piter_sg_next(struct vmw_piter *viter)
+{
+        return __sg_page_iter_next(&viter->iter);
+}
+
+
+/**
+ * Helper functions to return a pointer to the current page.
+ *
+ * @viter: Pointer to the iterator
+ *
+ * These functions return a pointer to the page currently
+ * pointed to by @viter. Functions are selected depending on the
+ * current mapping mode.
+ */
+static struct page *__vmw_piter_non_sg_page(struct vmw_piter *viter)
+{
+        return viter->pages[viter->i];
+}
+
+static struct page *__vmw_piter_sg_page(struct vmw_piter *viter)
+{
+        return sg_page_iter_page(&viter->iter);
+}
+
+
+/**
+ * Helper functions to return the DMA address of the current page.
+ *
+ * @viter: Pointer to the iterator
+ *
+ * These functions return the DMA address of the page currently
+ * pointed to by @viter. Functions are selected depending on the
+ * current mapping mode.
+ */
+static dma_addr_t __vmw_piter_phys_addr(struct vmw_piter *viter)
+{
+        return page_to_phys(viter->pages[viter->i]);
+}
+
+static dma_addr_t __vmw_piter_dma_addr(struct vmw_piter *viter)
+{
+        return viter->addrs[viter->i];
+}
+
+static dma_addr_t __vmw_piter_sg_addr(struct vmw_piter *viter)
+{
+        return sg_page_iter_dma_address(&viter->iter);
+}
+
+
+/**
+ * vmw_piter_start - Initialize a struct vmw_piter.
+ *
+ * @viter: Pointer to the iterator to initialize
+ * @vsgt: Pointer to a struct vmw_sg_table to initialize from
+ *
+ * Note that we're following the convention of __sg_page_iter_start, so that
+ * the iterator doesn't point to a valid page after initialization; it has
+ * to be advanced one step first.
+ */
+void vmw_piter_start(struct vmw_piter *viter, const struct vmw_sg_table *vsgt,
+                     unsigned long p_offset)
+{
+        viter->i = p_offset - 1;
+        viter->num_pages = vsgt->num_pages;
+        switch (vsgt->mode) {
+        case vmw_dma_phys:
+                viter->next = &__vmw_piter_non_sg_next;
+                viter->dma_address = &__vmw_piter_phys_addr;
+                viter->page = &__vmw_piter_non_sg_page;
+                viter->pages = vsgt->pages;
+                break;
+        case vmw_dma_alloc_coherent:
+                viter->next = &__vmw_piter_non_sg_next;
+                viter->dma_address = &__vmw_piter_dma_addr;
+                viter->page = &__vmw_piter_non_sg_page;
+                viter->addrs = vsgt->addrs;
+                break;
+        case vmw_dma_map_populate:
+        case vmw_dma_map_bind:
+                viter->next = &__vmw_piter_sg_next;
+                viter->dma_address = &__vmw_piter_sg_addr;
+                viter->page = &__vmw_piter_sg_page;
+                __sg_page_iter_start(&viter->iter, vsgt->sgt->sgl,
+                                     vsgt->sgt->orig_nents, p_offset);
+                break;
+        default:
+                BUG();
+        }
+}
+
+/**
+ * vmw_ttm_unmap_from_dma - unmap  device addresses previsouly mapped for
+ * TTM pages
+ *
+ * @vmw_tt: Pointer to a struct vmw_ttm_backend
+ *
+ * Used to free dma mappings previously mapped by vmw_ttm_map_for_dma.
+ */
+static void vmw_ttm_unmap_from_dma(struct vmw_ttm_tt *vmw_tt)
+{
+        struct device *dev = vmw_tt->dev_priv->dev->dev;
+
+        dma_unmap_sg(dev, vmw_tt->sgt.sgl, vmw_tt->sgt.nents,
+                DMA_BIDIRECTIONAL);
+        vmw_tt->sgt.nents = vmw_tt->sgt.orig_nents;
+}
+
+/**
+ * vmw_ttm_map_for_dma - map TTM pages to get device addresses
+ *
+ * @vmw_tt: Pointer to a struct vmw_ttm_backend
+ *
+ * This function is used to get device addresses from the kernel DMA layer.
+ * However, it's violating the DMA API in that when this operation has been
+ * performed, it's illegal for the CPU to write to the pages without first
+ * unmapping the DMA mappings, or calling dma_sync_sg_for_cpu(). It is
+ * therefore only legal to call this function if we know that the function
+ * dma_sync_sg_for_cpu() is a NOP, and dma_sync_sg_for_device() is at most
+ * a CPU write buffer flush.
+ */
+static int vmw_ttm_map_for_dma(struct vmw_ttm_tt *vmw_tt)
+{
+        struct device *dev = vmw_tt->dev_priv->dev->dev;
+        int ret;
+
+        ret = dma_map_sg(dev, vmw_tt->sgt.sgl, vmw_tt->sgt.orig_nents,
+                         DMA_BIDIRECTIONAL);
+        if (unlikely(ret == 0))
+                return -ENOMEM;
+
+        vmw_tt->sgt.nents = ret;
+
+        return 0;
+}
+
+/**
+ * vmw_ttm_map_dma - Make sure TTM pages are visible to the device
+ *
+ * @vmw_tt: Pointer to a struct vmw_ttm_tt
+ *
+ * Select the correct function for and make sure the TTM pages are
+ * visible to the device. Allocate storage for the device mappings.
+ * If a mapping has already been performed, indicated by the storage
+ * pointer being non NULL, the function returns success.
+ */
+static int vmw_ttm_map_dma(struct vmw_ttm_tt *vmw_tt)
+{
+        struct vmw_private *dev_priv = vmw_tt->dev_priv;
+        struct ttm_mem_global *glob = vmw_mem_glob(dev_priv);
+        struct vmw_sg_table *vsgt = &vmw_tt->vsgt;
+        struct vmw_piter iter;
+        dma_addr_t old;
+        int ret = 0;
+        static size_t sgl_size;
+        static size_t sgt_size;
+
+        if (vmw_tt->mapped)
+                return 0;
+
+        vsgt->mode = dev_priv->map_mode;
+        vsgt->pages = vmw_tt->dma_ttm.ttm.pages;
+        vsgt->num_pages = vmw_tt->dma_ttm.ttm.num_pages;
+        vsgt->addrs = vmw_tt->dma_ttm.dma_address;
+        vsgt->sgt = &vmw_tt->sgt;
+
+        switch (dev_priv->map_mode) {
+        case vmw_dma_map_bind:
+        case vmw_dma_map_populate:
+                if (unlikely(!sgl_size)) {
+                        sgl_size = ttm_round_pot(sizeof(struct scatterlist));
+                        sgt_size = ttm_round_pot(sizeof(struct sg_table));
+                }
+                vmw_tt->sg_alloc_size = sgt_size + sgl_size * vsgt->num_pages;
+                ret = ttm_mem_global_alloc(glob, vmw_tt->sg_alloc_size, false,
+                                           true);
+                if (unlikely(ret != 0))
+                        return ret;
+
+                ret = sg_alloc_table_from_pages(&vmw_tt->sgt, vsgt->pages,
+                                                vsgt->num_pages, 0,
+                                                (unsigned long)
+                                                vsgt->num_pages << PAGE_SHIFT,
+                                                GFP_KERNEL);
+                if (unlikely(ret != 0))
+                        goto out_sg_alloc_fail;
+
+                if (vsgt->num_pages > vmw_tt->sgt.nents) {
+                        uint64_t over_alloc =
+                                sgl_size * (vsgt->num_pages -
+                                            vmw_tt->sgt.nents);
+
+                        ttm_mem_global_free(glob, over_alloc);
+                        vmw_tt->sg_alloc_size -= over_alloc;
+                }
+
+                ret = vmw_ttm_map_for_dma(vmw_tt);
+                if (unlikely(ret != 0))
+                        goto out_map_fail;
+
+                break;
+        default:
+                break;
+        }
+
+        old = ~((dma_addr_t) 0);
+        vmw_tt->vsgt.num_regions = 0;
+        for (vmw_piter_start(&iter, vsgt, 0); vmw_piter_next(&iter);) {
+                dma_addr_t cur = vmw_piter_dma_addr(&iter);
+
+                if (cur != old + PAGE_SIZE)
+                        vmw_tt->vsgt.num_regions++;
+                old = cur;
+        }
+
+        vmw_tt->mapped = true;
+        return 0;
+
+out_map_fail:
+        sg_free_table(vmw_tt->vsgt.sgt);
+        vmw_tt->vsgt.sgt = NULL;
+out_sg_alloc_fail:
+        ttm_mem_global_free(glob, vmw_tt->sg_alloc_size);
+        return ret;
+}
+
+/**
+ * vmw_ttm_unmap_dma - Tear down any TTM page device mappings
+ *
+ * @vmw_tt: Pointer to a struct vmw_ttm_tt
+ *
+ * Tear down any previously set up device DMA mappings and free
+ * any storage space allocated for them. If there are no mappings set up,
+ * this function is a NOP.
+ */
+static void vmw_ttm_unmap_dma(struct vmw_ttm_tt *vmw_tt)
+{
+        struct vmw_private *dev_priv = vmw_tt->dev_priv;
+
+        if (!vmw_tt->vsgt.sgt)
+                return;
+
+        switch (dev_priv->map_mode) {
+        case vmw_dma_map_bind:
+        case vmw_dma_map_populate:
+                vmw_ttm_unmap_from_dma(vmw_tt);
+                sg_free_table(vmw_tt->vsgt.sgt);
+                vmw_tt->vsgt.sgt = NULL;
+                ttm_mem_global_free(vmw_mem_glob(dev_priv),
+                                    vmw_tt->sg_alloc_size);
+                break;
+        default:
+                break;
+        }
+        vmw_tt->mapped = false;
+}
+
 static int vmw_ttm_bind(struct ttm_tt *ttm, struct ttm_mem_reg *bo_mem)
 {
-        struct vmw_ttm_tt *vmw_be = container_of(ttm, struct vmw_ttm_tt, ttm);
+        struct vmw_ttm_tt *vmw_be =
+                container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
+        int ret;
+
+        ret = vmw_ttm_map_dma(vmw_be);
+        if (unlikely(ret != 0))
+                return ret;
 
         vmw_be->gmr_id = bo_mem->start;
 
-        return vmw_gmr_bind(vmw_be->dev_priv, ttm->pages,
+        return vmw_gmr_bind(vmw_be->dev_priv, &vmw_be->vsgt,
                             ttm->num_pages, vmw_be->gmr_id);
 }
 
 static int vmw_ttm_unbind(struct ttm_tt *ttm)
 {
-        struct vmw_ttm_tt *vmw_be = container_of(ttm, struct vmw_ttm_tt, ttm);
+        struct vmw_ttm_tt *vmw_be =
+                container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
 
         vmw_gmr_unbind(vmw_be->dev_priv, vmw_be->gmr_id);
+
+        if (vmw_be->dev_priv->map_mode == vmw_dma_map_bind)
+                vmw_ttm_unmap_dma(vmw_be);
+
         return 0;
 }
 
 static void vmw_ttm_destroy(struct ttm_tt *ttm)
 {
-        struct vmw_ttm_tt *vmw_be = container_of(ttm, struct vmw_ttm_tt, ttm);
-
-        ttm_tt_fini(ttm);
+        struct vmw_ttm_tt *vmw_be =
+                container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
+
+        vmw_ttm_unmap_dma(vmw_be);
+        if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent)
+                ttm_dma_tt_fini(&vmw_be->dma_ttm);
+        else
+                ttm_tt_fini(ttm);
         kfree(vmw_be);
 }
 
+static int vmw_ttm_populate(struct ttm_tt *ttm)
+{
+        struct vmw_ttm_tt *vmw_tt =
+                container_of(ttm, struct vmw_ttm_tt, dma_ttm.ttm);
+        struct vmw_private *dev_priv = vmw_tt->dev_priv;
+        struct ttm_mem_global *glob = vmw_mem_glob(dev_priv);
+        int ret;
+
+        if (ttm->state != tt_unpopulated)
+                return 0;
+
+        if (dev_priv->map_mode == vmw_dma_alloc_coherent) {
+                size_t size =
+                        ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t));
+                ret = ttm_mem_global_alloc(glob, size, false, true);
+                if (unlikely(ret != 0))
+                        return ret;
+
+                ret = ttm_dma_populate(&vmw_tt->dma_ttm, dev_priv->dev->dev);
+                if (unlikely(ret != 0))
+                        ttm_mem_global_free(glob, size);
+        } else
+                ret = ttm_pool_populate(ttm);
+
+        return ret;
+}
+
+static void vmw_ttm_unpopulate(struct ttm_tt *ttm)
+{
+        struct vmw_ttm_tt *vmw_tt = container_of(ttm, struct vmw_ttm_tt,
+                                                 dma_ttm.ttm);
+        struct vmw_private *dev_priv = vmw_tt->dev_priv;
+        struct ttm_mem_global *glob = vmw_mem_glob(dev_priv);
+
+        vmw_ttm_unmap_dma(vmw_tt);
+        if (dev_priv->map_mode == vmw_dma_alloc_coherent) {
+                size_t size =
+                        ttm_round_pot(ttm->num_pages * sizeof(dma_addr_t));
+
+                ttm_dma_unpopulate(&vmw_tt->dma_ttm, dev_priv->dev->dev);
+                ttm_mem_global_free(glob, size);
+        } else
+                ttm_pool_unpopulate(ttm);
+}
+
 static struct ttm_backend_func vmw_ttm_func = {
         .bind = vmw_ttm_bind,
         .unbind = vmw_ttm_unbind,
@@ -183,20 +520,28 @@ struct ttm_tt *vmw_ttm_tt_create(struct ttm_bo_device *bdev,
                                  struct page *dummy_read_page)
 {
         struct vmw_ttm_tt *vmw_be;
+        int ret;
 
-        vmw_be = kmalloc(sizeof(*vmw_be), GFP_KERNEL);
+        vmw_be = kzalloc(sizeof(*vmw_be), GFP_KERNEL);
         if (!vmw_be)
                 return NULL;
 
-        vmw_be->ttm.func = &vmw_ttm_func;
+        vmw_be->dma_ttm.ttm.func = &vmw_ttm_func;
         vmw_be->dev_priv = container_of(bdev, struct vmw_private, bdev);
 
-        if (ttm_tt_init(&vmw_be->ttm, bdev, size, page_flags, dummy_read_page)) {
-                kfree(vmw_be);
-                return NULL;
-        }
-
-        return &vmw_be->ttm;
+        if (vmw_be->dev_priv->map_mode == vmw_dma_alloc_coherent)
+                ret = ttm_dma_tt_init(&vmw_be->dma_ttm, bdev, size, page_flags,
+                                      dummy_read_page);
+        else
+                ret = ttm_tt_init(&vmw_be->dma_ttm.ttm, bdev, size, page_flags,
+                                  dummy_read_page);
+        if (unlikely(ret != 0))
+                goto out_no_init;
+
+        return &vmw_be->dma_ttm.ttm;
+out_no_init:
+        kfree(vmw_be);
+        return NULL;
 }
 
 int vmw_invalidate_caches(struct ttm_bo_device *bdev, uint32_t flags)
@@ -332,8 +677,8 @@ static int vmw_sync_obj_wait(void *sync_obj, bool lazy, bool interruptible)
 
 struct ttm_bo_driver vmw_bo_driver = {
         .ttm_tt_create = &vmw_ttm_tt_create,
-        .ttm_tt_populate = &ttm_pool_populate,
-        .ttm_tt_unpopulate = &ttm_pool_unpopulate,
+        .ttm_tt_populate = &vmw_ttm_populate,
+        .ttm_tt_unpopulate = &vmw_ttm_unpopulate,
         .invalidate_caches = vmw_invalidate_caches,
         .init_mem_type = vmw_init_mem_type,
         .evict_flags = vmw_evict_flags,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index 0508f93b9795..20d5485eaf98 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -32,6 +32,7 @@
 #include <drm/ttm/ttm_bo_driver.h>
 #include <drm/ttm/ttm_object.h>
 #include <drm/ttm/ttm_module.h>
+#include <linux/dma_remapping.h>
 
 #define VMWGFX_DRIVER_NAME "vmwgfx"
 #define VMWGFX_DRIVER_DESC "Linux drm driver for VMware graphics devices"
@@ -185,6 +186,9 @@ static struct pci_device_id vmw_pci_id_list[] = {
 MODULE_DEVICE_TABLE(pci, vmw_pci_id_list);
 
 static int enable_fbdev = IS_ENABLED(CONFIG_DRM_VMWGFX_FBCON);
+static int vmw_force_iommu;
+static int vmw_restrict_iommu;
+static int vmw_force_coherent;
 
 static int vmw_probe(struct pci_dev *, const struct pci_device_id *);
 static void vmw_master_init(struct vmw_master *);
@@ -193,6 +197,13 @@ static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val,
 
 MODULE_PARM_DESC(enable_fbdev, "Enable vmwgfx fbdev");
 module_param_named(enable_fbdev, enable_fbdev, int, 0600);
+MODULE_PARM_DESC(force_dma_api, "Force using the DMA API for TTM pages");
+module_param_named(force_dma_api, vmw_force_iommu, int, 0600);
+MODULE_PARM_DESC(restrict_iommu, "Try to limit IOMMU usage for TTM pages");
+module_param_named(restrict_iommu, vmw_restrict_iommu, int, 0600);
+MODULE_PARM_DESC(force_coherent, "Force coherent TTM pages");
+module_param_named(force_coherent, vmw_force_coherent, int, 0600);
+
 
 static void vmw_print_capabilities(uint32_t capabilities)
 {
@@ -427,12 +438,85 @@ static void vmw_get_initial_size(struct vmw_private *dev_priv)
         dev_priv->initial_height = height;
 }
 
+/**
+ * vmw_dma_select_mode - Determine how DMA mappings should be set up for this
+ * system.
+ *
+ * @dev_priv: Pointer to a struct vmw_private
+ *
+ * This functions tries to determine the IOMMU setup and what actions
+ * need to be taken by the driver to make system pages visible to the
+ * device.
+ * If this function decides that DMA is not possible, it returns -EINVAL.
+ * The driver may then try to disable features of the device that require
+ * DMA.
+ */
+static int vmw_dma_select_mode(struct vmw_private *dev_priv)
+{
+        static const char *names[vmw_dma_map_max] = {
+                [vmw_dma_phys] = "Using physical TTM page addresses.",
+                [vmw_dma_alloc_coherent] = "Using coherent TTM pages.",
+                [vmw_dma_map_populate] = "Keeping DMA mappings.",
+                [vmw_dma_map_bind] = "Giving up DMA mappings early."};
+#ifdef CONFIG_X86
+        const struct dma_map_ops *dma_ops = get_dma_ops(dev_priv->dev->dev);
+
+#ifdef CONFIG_INTEL_IOMMU
+        if (intel_iommu_enabled) {
+                dev_priv->map_mode = vmw_dma_map_populate;
+                goto out_fixup;
+        }
+#endif
+
+        if (!(vmw_force_iommu || vmw_force_coherent)) {
+                dev_priv->map_mode = vmw_dma_phys;
+                DRM_INFO("DMA map mode: %s\n", names[dev_priv->map_mode]);
+                return 0;
+        }
+
+        dev_priv->map_mode = vmw_dma_map_populate;
+
+        if (dma_ops->sync_single_for_cpu)
+                dev_priv->map_mode = vmw_dma_alloc_coherent;
+#ifdef CONFIG_SWIOTLB
+        if (swiotlb_nr_tbl() == 0)
+                dev_priv->map_mode = vmw_dma_map_populate;
+#endif
+
+#ifdef CONFIG_INTEL_IOMMU
+out_fixup:
+#endif
+        if (dev_priv->map_mode == vmw_dma_map_populate &&
+            vmw_restrict_iommu)
+                dev_priv->map_mode = vmw_dma_map_bind;
+
+        if (vmw_force_coherent)
+                dev_priv->map_mode = vmw_dma_alloc_coherent;
+
+#if !defined(CONFIG_SWIOTLB) && !defined(CONFIG_INTEL_IOMMU)
+        /*
+         * No coherent page pool
+         */
+        if (dev_priv->map_mode == vmw_dma_alloc_coherent)
+                return -EINVAL;
+#endif
+
+#else /* CONFIG_X86 */
+        dev_priv->map_mode = vmw_dma_map_populate;
+#endif /* CONFIG_X86 */
+
+        DRM_INFO("DMA map mode: %s\n", names[dev_priv->map_mode]);
+
+        return 0;
+}
+
 static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
 {
         struct vmw_private *dev_priv;
         int ret;
         uint32_t svga_id;
         enum vmw_res_type i;
+        bool refuse_dma = false;
 
         dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL);
         if (unlikely(dev_priv == NULL)) {
@@ -481,6 +565,11 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
         }
 
         dev_priv->capabilities = vmw_read(dev_priv, SVGA_REG_CAPABILITIES);
+        ret = vmw_dma_select_mode(dev_priv);
+        if (unlikely(ret != 0)) {
+                DRM_INFO("Restricting capabilities due to IOMMU setup.\n");
+                refuse_dma = true;
+        }
 
         dev_priv->vram_size = vmw_read(dev_priv, SVGA_REG_VRAM_SIZE);
         dev_priv->mmio_size = vmw_read(dev_priv, SVGA_REG_MEM_SIZE);
@@ -558,8 +647,9 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
         }
 
         dev_priv->has_gmr = true;
-        if (ttm_bo_init_mm(&dev_priv->bdev, VMW_PL_GMR,
-                           dev_priv->max_gmr_ids) != 0) {
+        if (((dev_priv->capabilities & (SVGA_CAP_GMR | SVGA_CAP_GMR2)) == 0) ||
+            refuse_dma || ttm_bo_init_mm(&dev_priv->bdev, VMW_PL_GMR,
+                                         dev_priv->max_gmr_ids) != 0) {
                 DRM_INFO("No GMR memory available. "
                          "Graphics memory resources are very limited.\n");
                 dev_priv->has_gmr = false;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index 150ec64af617..e401d5dbcb96 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -177,6 +177,58 @@ struct vmw_res_cache_entry {
         struct vmw_resource_val_node *node;
 };
 
+/**
+ * enum vmw_dma_map_mode - indicate how to perform TTM page dma mappings.
+ */
+enum vmw_dma_map_mode {
+        vmw_dma_phys,           /* Use physical page addresses */
+        vmw_dma_alloc_coherent, /* Use TTM coherent pages */
+        vmw_dma_map_populate,   /* Unmap from DMA just after unpopulate */
+        vmw_dma_map_bind,       /* Unmap from DMA just before unbind */
+        vmw_dma_map_max
+};
+
+/**
+ * struct vmw_sg_table - Scatter/gather table for binding, with additional
+ * device-specific information.
+ *
+ * @sgt: Pointer to a struct sg_table with binding information
+ * @num_regions: Number of regions with device-address contigous pages
+ */
+struct vmw_sg_table {
+        enum vmw_dma_map_mode mode;
+        struct page **pages;
+        const dma_addr_t *addrs;
+        struct sg_table *sgt;
+        unsigned long num_regions;
+        unsigned long num_pages;
+};
+
+/**
+ * struct vmw_piter - Page iterator that iterates over a list of pages
+ * and DMA addresses that could be either a scatter-gather list or
+ * arrays
+ *
+ * @pages: Array of page pointers to the pages.
+ * @addrs: DMA addresses to the pages if coherent pages are used.
+ * @iter: Scatter-gather page iterator. Current position in SG list.
+ * @i: Current position in arrays.
+ * @num_pages: Number of pages total.
+ * @next: Function to advance the iterator. Returns false if past the list
+ * of pages, true otherwise.
+ * @dma_address: Function to return the DMA address of the current page.
+ */
+struct vmw_piter {
+        struct page **pages;
+        const dma_addr_t *addrs;
+        struct sg_page_iter iter;
+        unsigned long i;
+        unsigned long num_pages;
+        bool (*next)(struct vmw_piter *);
+        dma_addr_t (*dma_address)(struct vmw_piter *);
+        struct page *(*page)(struct vmw_piter *);
+};
+
 struct vmw_sw_context{
         struct drm_open_hash res_ht;
         bool res_ht_initialized;
@@ -358,6 +410,11 @@ struct vmw_private {
 
         struct list_head res_lru[vmw_res_max];
         uint32_t used_memory_size;
+
+        /*
+         * DMA mapping stuff.
+         */
+        enum vmw_dma_map_mode map_mode;
 };
 
 static inline struct vmw_surface *vmw_res_to_srf(struct vmw_resource *res)
@@ -405,7 +462,7 @@ void vmw_3d_resource_dec(struct vmw_private *dev_priv, bool hide_svga);
  */
 
 extern int vmw_gmr_bind(struct vmw_private *dev_priv,
-                        struct page *pages[],
+                        const struct vmw_sg_table *vsgt,
                         unsigned long num_pages,
                         int gmr_id);
 extern void vmw_gmr_unbind(struct vmw_private *dev_priv, int gmr_id);
@@ -568,6 +625,45 @@ extern struct ttm_placement vmw_evictable_placement;
 extern struct ttm_placement vmw_srf_placement;
 extern struct ttm_bo_driver vmw_bo_driver;
 extern int vmw_dma_quiescent(struct drm_device *dev);
+extern void vmw_piter_start(struct vmw_piter *viter,
+                            const struct vmw_sg_table *vsgt,
+                            unsigned long p_offs);
+
+/**
+ * vmw_piter_next - Advance the iterator one page.
+ *
+ * @viter: Pointer to the iterator to advance.
+ *
+ * Returns false if past the list of pages, true otherwise.
+ */
+static inline bool vmw_piter_next(struct vmw_piter *viter)
+{
+        return viter->next(viter);
+}
+
+/**
+ * vmw_piter_dma_addr - Return the DMA address of the current page.
+ *
+ * @viter: Pointer to the iterator
+ *
+ * Returns the DMA address of the page pointed to by @viter.
+ */
+static inline dma_addr_t vmw_piter_dma_addr(struct vmw_piter *viter)
+{
+        return viter->dma_address(viter);
+}
+
+/**
+ * vmw_piter_page - Return a pointer to the current page.
+ *
+ * @viter: Pointer to the iterator
+ *
+ * Returns the DMA address of the page pointed to by @viter.
+ */
+static inline struct page *vmw_piter_page(struct vmw_piter *viter)
+{
+        return viter->page(viter);
+}
 
 /**
  * Command submission - vmwgfx_execbuf.c
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c b/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c
index 1a0bf07fe54b..6ef0b035becb 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_gmr.c
@@ -32,9 +32,11 @@
 #define VMW_PPN_SIZE (sizeof(unsigned long))
 /* A future safe maximum remap size. */
 #define VMW_PPN_PER_REMAP ((31 * 1024) / VMW_PPN_SIZE)
+#define DMA_ADDR_INVALID ((dma_addr_t) 0)
+#define DMA_PAGE_INVALID 0UL
 
 static int vmw_gmr2_bind(struct vmw_private *dev_priv,
-                         struct page *pages[],
+                         struct vmw_piter *iter,
                          unsigned long num_pages,
                          int gmr_id)
 {
@@ -81,11 +83,13 @@ static int vmw_gmr2_bind(struct vmw_private *dev_priv,
 
                 for (i = 0; i < nr; ++i) {
                         if (VMW_PPN_SIZE <= 4)
-                                *cmd = page_to_pfn(*pages++);
+                                *cmd = vmw_piter_dma_addr(iter) >> PAGE_SHIFT;
                         else
-                                *((uint64_t *)cmd) = page_to_pfn(*pages++);
+                                *((uint64_t *)cmd) = vmw_piter_dma_addr(iter) >>
+                                        PAGE_SHIFT;
 
                         cmd += VMW_PPN_SIZE / sizeof(*cmd);
+                        vmw_piter_next(iter);
                 }
 
                 num_pages -= nr;
@@ -120,22 +124,56 @@ static void vmw_gmr2_unbind(struct vmw_private *dev_priv,
         vmw_fifo_commit(dev_priv, define_size);
 }
 
+
+static void vmw_gmr_free_descriptors(struct device *dev, dma_addr_t desc_dma,
+                                     struct list_head *desc_pages)
+{
+        struct page *page, *next;
+        struct svga_guest_mem_descriptor *page_virtual;
+        unsigned int desc_per_page = PAGE_SIZE /
+                sizeof(struct svga_guest_mem_descriptor) - 1;
+
+        if (list_empty(desc_pages))
+                return;
+
+        list_for_each_entry_safe(page, next, desc_pages, lru) {
+                list_del_init(&page->lru);
+
+                if (likely(desc_dma != DMA_ADDR_INVALID)) {
+                        dma_unmap_page(dev, desc_dma, PAGE_SIZE,
+                                       DMA_TO_DEVICE);
+                }
+
+                page_virtual = kmap_atomic(page);
+                desc_dma = (dma_addr_t)
+                        le32_to_cpu(page_virtual[desc_per_page].ppn) <<
+                        PAGE_SHIFT;
+                kunmap_atomic(page_virtual);
+
+                __free_page(page);
+        }
+}
+
 /**
  * FIXME: Adjust to the ttm lowmem / highmem storage to minimize
  * the number of used descriptors.
+ *
  */
 
-static int vmw_gmr_build_descriptors(struct list_head *desc_pages,
-                                     struct page *pages[],
-                                     unsigned long num_pages)
+static int vmw_gmr_build_descriptors(struct device *dev,
+                                     struct list_head *desc_pages,
+                                     struct vmw_piter *iter,
+                                     unsigned long num_pages,
+                                     dma_addr_t *first_dma)
 {
-        struct page *page, *next;
+        struct page *page;
         struct svga_guest_mem_descriptor *page_virtual = NULL;
         struct svga_guest_mem_descriptor *desc_virtual = NULL;
         unsigned int desc_per_page;
         unsigned long prev_pfn;
         unsigned long pfn;
         int ret;
+        dma_addr_t desc_dma;
 
         desc_per_page = PAGE_SIZE /
             sizeof(struct svga_guest_mem_descriptor) - 1;
@@ -148,23 +186,12 @@ static int vmw_gmr_build_descriptors(struct list_head *desc_pages,
                 }
 
                 list_add_tail(&page->lru, desc_pages);
-
-                /*
-                 * Point previous page terminating descriptor to this
-                 * page before unmapping it.
-                 */
-
-                if (likely(page_virtual != NULL)) {
-                        desc_virtual->ppn = page_to_pfn(page);
-                        kunmap_atomic(page_virtual);
-                }
-
                 page_virtual = kmap_atomic(page);
                 desc_virtual = page_virtual - 1;
                 prev_pfn = ~(0UL);
 
                 while (likely(num_pages != 0)) {
-                        pfn = page_to_pfn(*pages);
+                        pfn = vmw_piter_dma_addr(iter) >> PAGE_SHIFT;
 
                         if (pfn != prev_pfn + 1) {
 
@@ -181,104 +208,82 @@ static int vmw_gmr_build_descriptors(struct list_head *desc_pages,
                         }
                         prev_pfn = pfn;
                         --num_pages;
-                        ++pages;
+                        vmw_piter_next(iter);
                 }
 
-                (++desc_virtual)->ppn = cpu_to_le32(0);
+                (++desc_virtual)->ppn = DMA_PAGE_INVALID;
                 desc_virtual->num_pages = cpu_to_le32(0);
+                kunmap_atomic(page_virtual);
         }
 
-        if (likely(page_virtual != NULL))
+        desc_dma = 0;
+        list_for_each_entry_reverse(page, desc_pages, lru) {
+                page_virtual = kmap_atomic(page);
+                page_virtual[desc_per_page].ppn = cpu_to_le32
+                        (desc_dma >> PAGE_SHIFT);
                 kunmap_atomic(page_virtual);
+                desc_dma = dma_map_page(dev, page, 0, PAGE_SIZE,
+                                        DMA_TO_DEVICE);
+
+                if (unlikely(dma_mapping_error(dev, desc_dma)))
+                        goto out_err;
+        }
+        *first_dma = desc_dma;
 
         return 0;
 out_err:
-        list_for_each_entry_safe(page, next, desc_pages, lru) {
-                list_del_init(&page->lru);
-                __free_page(page);
-        }
+        vmw_gmr_free_descriptors(dev, DMA_ADDR_INVALID, desc_pages);
         return ret;
 }
 
-static inline void vmw_gmr_free_descriptors(struct list_head *desc_pages)
-{
-        struct page *page, *next;
-
-        list_for_each_entry_safe(page, next, desc_pages, lru) {
-                list_del_init(&page->lru);
-                __free_page(page);
-        }
-}
-
 static void vmw_gmr_fire_descriptors(struct vmw_private *dev_priv,
-                                     int gmr_id, struct list_head *desc_pages)
+                                     int gmr_id, dma_addr_t desc_dma)
 {
-        struct page *page;
-
-        if (unlikely(list_empty(desc_pages)))
-                return;
-
-        page = list_entry(desc_pages->next, struct page, lru);
-
         mutex_lock(&dev_priv->hw_mutex);
 
         vmw_write(dev_priv, SVGA_REG_GMR_ID, gmr_id);
         wmb();
-        vmw_write(dev_priv, SVGA_REG_GMR_DESCRIPTOR, page_to_pfn(page));
+        vmw_write(dev_priv, SVGA_REG_GMR_DESCRIPTOR, desc_dma >> PAGE_SHIFT);
         mb();
 
         mutex_unlock(&dev_priv->hw_mutex);
 
 }
 
-/**
- * FIXME: Adjust to the ttm lowmem / highmem storage to minimize
- * the number of used descriptors.
- */
-
-static unsigned long vmw_gmr_count_descriptors(struct page *pages[],
-                                        unsigned long num_pages)
-{
-        unsigned long prev_pfn = ~(0UL);
-        unsigned long pfn;
-        unsigned long descriptors = 0;
-
-        while (num_pages--) {
-                pfn = page_to_pfn(*pages++);
-                if (prev_pfn + 1 != pfn)
-                        ++descriptors;
-                prev_pfn = pfn;
-        }
-
-        return descriptors;
-}
-
 int vmw_gmr_bind(struct vmw_private *dev_priv,
-                 struct page *pages[],
+                 const struct vmw_sg_table *vsgt,
                  unsigned long num_pages,
                  int gmr_id)
 {
         struct list_head desc_pages;
+        dma_addr_t desc_dma = 0;
+        struct device *dev = dev_priv->dev->dev;
+        struct vmw_piter data_iter;
         int ret;
 
+        vmw_piter_start(&data_iter, vsgt, 0);
+
+        if (unlikely(!vmw_piter_next(&data_iter)))
+                return 0;
+
         if (likely(dev_priv->capabilities & SVGA_CAP_GMR2))
-                return vmw_gmr2_bind(dev_priv, pages, num_pages, gmr_id);
+                return vmw_gmr2_bind(dev_priv, &data_iter, num_pages, gmr_id);
 
         if (unlikely(!(dev_priv->capabilities & SVGA_CAP_GMR)))
                 return -EINVAL;
 
-        if (vmw_gmr_count_descriptors(pages, num_pages) >
-            dev_priv->max_gmr_descriptors)
+        if (vsgt->num_regions > dev_priv->max_gmr_descriptors)
                 return -EINVAL;
 
         INIT_LIST_HEAD(&desc_pages);
 
-        ret = vmw_gmr_build_descriptors(&desc_pages, pages, num_pages);
+        ret = vmw_gmr_build_descriptors(dev, &desc_pages, &data_iter,
+                                        num_pages, &desc_dma);
         if (unlikely(ret != 0))
                 return ret;
 
-        vmw_gmr_fire_descriptors(dev_priv, gmr_id, &desc_pages);
-        vmw_gmr_free_descriptors(&desc_pages);
+        vmw_gmr_fire_descriptors(dev_priv, gmr_id, desc_dma);
+        vmw_gmr_free_descriptors(dev, desc_dma, &desc_pages);
 
         return 0;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
index c509d40c4897..a51f48e3e917 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
@@ -168,7 +168,7 @@ int vmw_present_ioctl(struct drm_device *dev, void *data,
         fb = drm_framebuffer_lookup(dev, arg->fb_id);
         if (!fb) {
                 DRM_ERROR("Invalid framebuffer id.\n");
-                ret = -EINVAL;
+                ret = -ENOENT;
                 goto out_no_fb;
         }
         vfb = vmw_framebuffer_to_vfb(fb);
@@ -252,7 +252,7 @@ int vmw_present_readback_ioctl(struct drm_device *dev, void *data,
         fb = drm_framebuffer_lookup(dev, arg->fb_id);
         if (!fb) {
                 DRM_ERROR("Invalid framebuffer id.\n");
-                ret = -EINVAL;
+                ret = -ENOENT;
                 goto out_no_fb;
         }
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index fc43c0601236..ecb3d867b426 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -1508,7 +1508,7 @@ int vmw_kms_cursor_bypass_ioctl(struct drm_device *dev, void *data,
 
         obj = drm_mode_object_find(dev, arg->crtc_id, DRM_MODE_OBJECT_CRTC);
         if (!obj) {
-                ret = -EINVAL;
+                ret = -ENOENT;
                 goto out;
         }
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
index 37fb4befec82..252501a54def 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
@@ -32,6 +32,8 @@
 #include <drm/drmP.h>
 #include "vmwgfx_resource_priv.h"
 
+#define VMW_RES_EVICT_ERR_COUNT 10
+
 struct vmw_user_dma_buffer {
         struct ttm_base_object base;
         struct vmw_dma_buffer dma;
@@ -1091,8 +1093,9 @@ vmw_resource_backoff_reservation(struct ww_acquire_ctx *ticket,
  *                         to a backup buffer.
  *
  * @res:            The resource to evict.
+ * @interruptible:  Whether to wait interruptible.
  */
-int vmw_resource_do_evict(struct vmw_resource *res)
+int vmw_resource_do_evict(struct vmw_resource *res, bool interruptible)
 {
         struct ttm_validate_buffer val_buf;
         const struct vmw_res_func *func = res->func;
@@ -1102,7 +1105,8 @@ int vmw_resource_do_evict(struct vmw_resource *res)
         BUG_ON(!func->may_evict);
 
         val_buf.bo = NULL;
-        ret = vmw_resource_check_buffer(res, &ticket, true, &val_buf);
+        ret = vmw_resource_check_buffer(res, &ticket, interruptible,
+                                        &val_buf);
         if (unlikely(ret != 0))
                 return ret;
 
@@ -1141,6 +1145,7 @@ int vmw_resource_validate(struct vmw_resource *res)
         struct vmw_private *dev_priv = res->dev_priv;
         struct list_head *lru_list = &dev_priv->res_lru[res->func->res_type];
         struct ttm_validate_buffer val_buf;
+        unsigned err_count = 0;
 
         if (likely(!res->func->may_evict))
                 return 0;
@@ -1155,7 +1160,7 @@ int vmw_resource_validate(struct vmw_resource *res)
 
                 write_lock(&dev_priv->resource_lock);
                 if (list_empty(lru_list) || !res->func->may_evict) {
-                        DRM_ERROR("Out of device device id entries "
+                        DRM_ERROR("Out of device device resources "
                                   "for %s.\n", res->func->type_name);
                         ret = -EBUSY;
                         write_unlock(&dev_priv->resource_lock);
@@ -1168,7 +1173,19 @@ int vmw_resource_validate(struct vmw_resource *res)
                 list_del_init(&evict_res->lru_head);
 
                 write_unlock(&dev_priv->resource_lock);
-                vmw_resource_do_evict(evict_res);
+
+                ret = vmw_resource_do_evict(evict_res, true);
+                if (unlikely(ret != 0)) {
+                        write_lock(&dev_priv->resource_lock);
+                        list_add_tail(&evict_res->lru_head, lru_list);
+                        write_unlock(&dev_priv->resource_lock);
+                        if (ret == -ERESTARTSYS ||
+                            ++err_count > VMW_RES_EVICT_ERR_COUNT) {
+                                vmw_resource_unreference(&evict_res);
+                                goto out_no_validate;
+                        }
+                }
+
                 vmw_resource_unreference(&evict_res);
         } while (1);
 
@@ -1253,13 +1270,15 @@ bool vmw_resource_needs_backup(const struct vmw_resource *res)
  * @type:           The resource type to evict
  *
  * To avoid thrashing starvation or as part of the hibernation sequence,
- * evict all evictable resources of a specific type.
+ * try to evict all evictable resources of a specific type.
  */
 static void vmw_resource_evict_type(struct vmw_private *dev_priv,
                                     enum vmw_res_type type)
 {
         struct list_head *lru_list = &dev_priv->res_lru[type];
         struct vmw_resource *evict_res;
+        unsigned err_count = 0;
+        int ret;
 
         do {
                 write_lock(&dev_priv->resource_lock);
@@ -1272,7 +1291,18 @@ static void vmw_resource_evict_type(struct vmw_private *dev_priv,
                                          lru_head));
                 list_del_init(&evict_res->lru_head);
                 write_unlock(&dev_priv->resource_lock);
-                vmw_resource_do_evict(evict_res);
+
+                ret = vmw_resource_do_evict(evict_res, false);
+                if (unlikely(ret != 0)) {
+                        write_lock(&dev_priv->resource_lock);
+                        list_add_tail(&evict_res->lru_head, lru_list);
+                        write_unlock(&dev_priv->resource_lock);
+                        if (++err_count > VMW_RES_EVICT_ERR_COUNT) {
+                                vmw_resource_unreference(&evict_res);
+                                return;
+                        }
+                }
+
                 vmw_resource_unreference(&evict_res);
         } while (1);
 
diff --git a/drivers/gpu/host1x/Kconfig b/drivers/gpu/host1x/Kconfig
index ccfd42b23606..7d6bed222542 100644
--- a/drivers/gpu/host1x/Kconfig
+++ b/drivers/gpu/host1x/Kconfig
@@ -19,6 +19,4 @@ config TEGRA_HOST1X_FIREWALL
 
           If unsure, choose Y.
 
-source "drivers/gpu/host1x/drm/Kconfig"
-
 endif
diff --git a/drivers/gpu/host1x/Makefile b/drivers/gpu/host1x/Makefile
index 3b037b6e0298..afa1e9e4e512 100644
--- a/drivers/gpu/host1x/Makefile
+++ b/drivers/gpu/host1x/Makefile
@@ -1,6 +1,5 @@
-ccflags-y = -Idrivers/gpu/host1x
-
 host1x-y = \
+        bus.o \
         syncpt.o \
         dev.o \
         intr.o \
@@ -8,13 +7,7 @@ host1x-y = \
         channel.o \
         job.o \
         debug.o \
-        hw/host1x01.o
-
-ccflags-y += -Iinclude/drm
-ccflags-$(CONFIG_DRM_TEGRA_DEBUG) += -DDEBUG
+        hw/host1x01.o \
+        hw/host1x02.o
 
-host1x-$(CONFIG_DRM_TEGRA) += drm/drm.o drm/fb.o drm/dc.o
-host1x-$(CONFIG_DRM_TEGRA) += drm/output.o drm/rgb.o drm/hdmi.o
-host1x-$(CONFIG_DRM_TEGRA) += drm/gem.o
-host1x-$(CONFIG_DRM_TEGRA) += drm/gr2d.o
 obj-$(CONFIG_TEGRA_HOST1X) += host1x.o
diff --git a/drivers/gpu/host1x/bus.c b/drivers/gpu/host1x/bus.c
new file mode 100644
index 000000000000..509383f8be03
--- /dev/null
+++ b/drivers/gpu/host1x/bus.c
@@ -0,0 +1,550 @@
+/*
+ * Copyright (C) 2012 Avionic Design GmbH
+ * Copyright (C) 2012-2013, NVIDIA Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/host1x.h>
+#include <linux/of.h>
+#include <linux/slab.h>
+
+#include "dev.h"
+
+static DEFINE_MUTEX(clients_lock);
+static LIST_HEAD(clients);
+
+static DEFINE_MUTEX(drivers_lock);
+static LIST_HEAD(drivers);
+
+static DEFINE_MUTEX(devices_lock);
+static LIST_HEAD(devices);
+
+struct host1x_subdev {
+        struct host1x_client *client;
+        struct device_node *np;
+        struct list_head list;
+};
+
+/**
+ * host1x_subdev_add() - add a new subdevice with an associated device node
+ */
+static int host1x_subdev_add(struct host1x_device *device,
+                             struct device_node *np)
+{
+        struct host1x_subdev *subdev;
+
+        subdev = kzalloc(sizeof(*subdev), GFP_KERNEL);
+        if (!subdev)
+                return -ENOMEM;
+
+        INIT_LIST_HEAD(&subdev->list);
+        subdev->np = of_node_get(np);
+
+        mutex_lock(&device->subdevs_lock);
+        list_add_tail(&subdev->list, &device->subdevs);
+        mutex_unlock(&device->subdevs_lock);
+
+        return 0;
+}
+
+/**
+ * host1x_subdev_del() - remove subdevice
+ */
+static void host1x_subdev_del(struct host1x_subdev *subdev)
+{
+        list_del(&subdev->list);
+        of_node_put(subdev->np);
+        kfree(subdev);
+}
+
+/**
+ * host1x_device_parse_dt() - scan device tree and add matching subdevices
+ */
+static int host1x_device_parse_dt(struct host1x_device *device)
+{
+        struct device_node *np;
+        int err;
+
+        for_each_child_of_node(device->dev.parent->of_node, np) {
+                if (of_match_node(device->driver->subdevs, np) &&
+                    of_device_is_available(np)) {
+                        err = host1x_subdev_add(device, np);
+                        if (err < 0)
+                                return err;
+                }
+        }
+
+        return 0;
+}
+
+static void host1x_subdev_register(struct host1x_device *device,
+                                   struct host1x_subdev *subdev,
+                                   struct host1x_client *client)
+{
+        int err;
+
+        /*
+         * Move the subdevice to the list of active (registered) subdevices
+         * and associate it with a client. At the same time, associate the
+         * client with its parent device.
+         */
+        mutex_lock(&device->subdevs_lock);
+        mutex_lock(&device->clients_lock);
+        list_move_tail(&client->list, &device->clients);
+        list_move_tail(&subdev->list, &device->active);
+        client->parent = &device->dev;
+        subdev->client = client;
+        mutex_unlock(&device->clients_lock);
+        mutex_unlock(&device->subdevs_lock);
+
+        /*
+         * When all subdevices have been registered, the composite device is
+         * ready to be probed.
+         */
+        if (list_empty(&device->subdevs)) {
+                err = device->driver->probe(device);
+                if (err < 0)
+                        dev_err(&device->dev, "probe failed: %d\n", err);
+        }
+}
+
+static void __host1x_subdev_unregister(struct host1x_device *device,
+                                       struct host1x_subdev *subdev)
+{
+        struct host1x_client *client = subdev->client;
+        int err;
+
+        /*
+         * If all subdevices have been activated, we're about to remove the
+         * first active subdevice, so unload the driver first.
+         */
+        if (list_empty(&device->subdevs)) {
+                err = device->driver->remove(device);
+                if (err < 0)
+                        dev_err(&device->dev, "remove failed: %d\n", err);
+        }
+
+        /*
+         * Move the subdevice back to the list of idle subdevices and remove
+         * it from list of clients.
+         */
+        mutex_lock(&device->clients_lock);
+        subdev->client = NULL;
+        client->parent = NULL;
+        list_move_tail(&subdev->list, &device->subdevs);
+        /*
+         * XXX: Perhaps don't do this here, but rather explicitly remove it
+         * when the device is about to be deleted.
+         *
+         * This is somewhat complicated by the fact that this function is
+         * used to remove the subdevice when a client is unregistered but
+         * also when the composite device is about to be removed.
+         */
+        list_del_init(&client->list);
+        mutex_unlock(&device->clients_lock);
+}
+
+static void host1x_subdev_unregister(struct host1x_device *device,
+                                     struct host1x_subdev *subdev)
+{
+        mutex_lock(&device->subdevs_lock);
+        __host1x_subdev_unregister(device, subdev);
+        mutex_unlock(&device->subdevs_lock);
+}
+
+int host1x_device_init(struct host1x_device *device)
+{
+        struct host1x_client *client;
+        int err;
+
+        mutex_lock(&device->clients_lock);
+
+        list_for_each_entry(client, &device->clients, list) {
+                if (client->ops && client->ops->init) {
+                        err = client->ops->init(client);
+                        if (err < 0) {
+                                dev_err(&device->dev,
+                                        "failed to initialize %s: %d\n",
+                                        dev_name(client->dev), err);
+                                mutex_unlock(&device->clients_lock);
+                                return err;
+                        }
+                }
+        }
+
+        mutex_unlock(&device->clients_lock);
+
+        return 0;
+}
+
+int host1x_device_exit(struct host1x_device *device)
+{
+        struct host1x_client *client;
+        int err;
+
+        mutex_lock(&device->clients_lock);
+
+        list_for_each_entry_reverse(client, &device->clients, list) {
+                if (client->ops && client->ops->exit) {
+                        err = client->ops->exit(client);
+                        if (err < 0) {
+                                dev_err(&device->dev,
+                                        "failed to cleanup %s: %d\n",
+                                        dev_name(client->dev), err);
+                                mutex_unlock(&device->clients_lock);
+                                return err;
+                        }
+                }
+        }
+
+        mutex_unlock(&device->clients_lock);
+
+        return 0;
+}
+
+static int host1x_register_client(struct host1x *host1x,
+                                  struct host1x_client *client)
+{
+        struct host1x_device *device;
+        struct host1x_subdev *subdev;
+
+        mutex_lock(&host1x->devices_lock);
+
+        list_for_each_entry(device, &host1x->devices, list) {
+                list_for_each_entry(subdev, &device->subdevs, list) {
+                        if (subdev->np == client->dev->of_node) {
+                                host1x_subdev_register(device, subdev, client);
+                                mutex_unlock(&host1x->devices_lock);
+                                return 0;
+                        }
+                }
+        }
+
+        mutex_unlock(&host1x->devices_lock);
+        return -ENODEV;
+}
+
+static int host1x_unregister_client(struct host1x *host1x,
+                                    struct host1x_client *client)
+{
+        struct host1x_device *device, *dt;
+        struct host1x_subdev *subdev;
+
+        mutex_lock(&host1x->devices_lock);
+
+        list_for_each_entry_safe(device, dt, &host1x->devices, list) {
+                list_for_each_entry(subdev, &device->active, list) {
+                        if (subdev->client == client) {
+                                host1x_subdev_unregister(device, subdev);
+                                mutex_unlock(&host1x->devices_lock);
+                                return 0;
+                        }
+                }
+        }
+
+        mutex_unlock(&host1x->devices_lock);
+        return -ENODEV;
+}
+
+struct bus_type host1x_bus_type = {
+        .name = "host1x",
+};
+
+int host1x_bus_init(void)
+{
+        return bus_register(&host1x_bus_type);
+}
+
+void host1x_bus_exit(void)
+{
+        bus_unregister(&host1x_bus_type);
+}
+
+static void host1x_device_release(struct device *dev)
+{
+        struct host1x_device *device = to_host1x_device(dev);
+
+        kfree(device);
+}
+
+static int host1x_device_add(struct host1x *host1x,
+                             struct host1x_driver *driver)
+{
+        struct host1x_client *client, *tmp;
+        struct host1x_subdev *subdev;
+        struct host1x_device *device;
+        int err;
+
+        device = kzalloc(sizeof(*device), GFP_KERNEL);
+        if (!device)
+                return -ENOMEM;
+
+        mutex_init(&device->subdevs_lock);
+        INIT_LIST_HEAD(&device->subdevs);
+        INIT_LIST_HEAD(&device->active);
+        mutex_init(&device->clients_lock);
+        INIT_LIST_HEAD(&device->clients);
+        INIT_LIST_HEAD(&device->list);
+        device->driver = driver;
+
+        device->dev.coherent_dma_mask = host1x->dev->coherent_dma_mask;
+        device->dev.dma_mask = &device->dev.coherent_dma_mask;
+        device->dev.release = host1x_device_release;
+        dev_set_name(&device->dev, driver->name);
+        device->dev.bus = &host1x_bus_type;
+        device->dev.parent = host1x->dev;
+
+        err = device_register(&device->dev);
+        if (err < 0)
+                return err;
+
+        err = host1x_device_parse_dt(device);
+        if (err < 0) {
+                device_unregister(&device->dev);
+                return err;
+        }
+
+        mutex_lock(&host1x->devices_lock);
+        list_add_tail(&device->list, &host1x->devices);
+        mutex_unlock(&host1x->devices_lock);
+
+        mutex_lock(&clients_lock);
+
+        list_for_each_entry_safe(client, tmp, &clients, list) {
+                list_for_each_entry(subdev, &device->subdevs, list) {
+                        if (subdev->np == client->dev->of_node) {
+                                host1x_subdev_register(device, subdev, client);
+                                break;
+                        }
+                }
+        }
+
+        mutex_unlock(&clients_lock);
+
+        return 0;
+}
+
+/*
+ * Removes a device by first unregistering any subdevices and then removing
+ * itself from the list of devices.
+ *
+ * This function must be called with the host1x->devices_lock held.
+ */
+static void host1x_device_del(struct host1x *host1x,
+                              struct host1x_device *device)
+{
+        struct host1x_subdev *subdev, *sd;
+        struct host1x_client *client, *cl;
+
+        mutex_lock(&device->subdevs_lock);
+
+        /* unregister subdevices */
+        list_for_each_entry_safe(subdev, sd, &device->active, list) {
+                /*
+                 * host1x_subdev_unregister() will remove the client from
+                 * any lists, so we'll need to manually add it back to the
+                 * list of idle clients.
+                 *
+                 * XXX: Alternatively, perhaps don't remove the client from
+                 * any lists in host1x_subdev_unregister() and instead do
+                 * that explicitly from host1x_unregister_client()?
+                 */
+                client = subdev->client;
+
+                __host1x_subdev_unregister(device, subdev);
+
+                /* add the client to the list of idle clients */
+                mutex_lock(&clients_lock);
+                list_add_tail(&client->list, &clients);
+                mutex_unlock(&clients_lock);
+        }
+
+        /* remove subdevices */
+        list_for_each_entry_safe(subdev, sd, &device->subdevs, list)
+                host1x_subdev_del(subdev);
+
+        mutex_unlock(&device->subdevs_lock);
+
+        /* move clients to idle list */
+        mutex_lock(&clients_lock);
+        mutex_lock(&device->clients_lock);
+
+        list_for_each_entry_safe(client, cl, &device->clients, list)
+                list_move_tail(&client->list, &clients);
+
+        mutex_unlock(&device->clients_lock);
+        mutex_unlock(&clients_lock);
+
+        /* finally remove the device */
+        list_del_init(&device->list);
+        device_unregister(&device->dev);
+}
+
+static void host1x_attach_driver(struct host1x *host1x,
+                                 struct host1x_driver *driver)
+{
+        struct host1x_device *device;
+        int err;
+
+        mutex_lock(&host1x->devices_lock);
+
+        list_for_each_entry(device, &host1x->devices, list) {
+                if (device->driver == driver) {
+                        mutex_unlock(&host1x->devices_lock);
+                        return;
+                }
+        }
+
+        mutex_unlock(&host1x->devices_lock);
+
+        err = host1x_device_add(host1x, driver);
+        if (err < 0)
+                dev_err(host1x->dev, "failed to allocate device: %d\n", err);
+}
+
+static void host1x_detach_driver(struct host1x *host1x,
+                                 struct host1x_driver *driver)
+{
+        struct host1x_device *device, *tmp;
+
+        mutex_lock(&host1x->devices_lock);
+
+        list_for_each_entry_safe(device, tmp, &host1x->devices, list)
+                if (device->driver == driver)
+                        host1x_device_del(host1x, device);
+
+        mutex_unlock(&host1x->devices_lock);
+}
+
+int host1x_register(struct host1x *host1x)
+{
+        struct host1x_driver *driver;
+
+        mutex_lock(&devices_lock);
+        list_add_tail(&host1x->list, &devices);
+        mutex_unlock(&devices_lock);
+
+        mutex_lock(&drivers_lock);
+
+        list_for_each_entry(driver, &drivers, list)
+                host1x_attach_driver(host1x, driver);
+
+        mutex_unlock(&drivers_lock);
+
+        return 0;
+}
+
+int host1x_unregister(struct host1x *host1x)
+{
+        struct host1x_driver *driver;
+
+        mutex_lock(&drivers_lock);
+
+        list_for_each_entry(driver, &drivers, list)
+                host1x_detach_driver(host1x, driver);
+
+        mutex_unlock(&drivers_lock);
+
+        mutex_lock(&devices_lock);
+        list_del_init(&host1x->list);
+        mutex_unlock(&devices_lock);
+
+        return 0;
+}
+
+int host1x_driver_register(struct host1x_driver *driver)
+{
+        struct host1x *host1x;
+
+        INIT_LIST_HEAD(&driver->list);
+
+        mutex_lock(&drivers_lock);
+        list_add_tail(&driver->list, &drivers);
+        mutex_unlock(&drivers_lock);
+
+        mutex_lock(&devices_lock);
+
+        list_for_each_entry(host1x, &devices, list)
+                host1x_attach_driver(host1x, driver);
+
+        mutex_unlock(&devices_lock);
+
+        return 0;
+}
+EXPORT_SYMBOL(host1x_driver_register);
+
+void host1x_driver_unregister(struct host1x_driver *driver)
+{
+        mutex_lock(&drivers_lock);
+        list_del_init(&driver->list);
+        mutex_unlock(&drivers_lock);
+}
+EXPORT_SYMBOL(host1x_driver_unregister);
+
+int host1x_client_register(struct host1x_client *client)
+{
+        struct host1x *host1x;
+        int err;
+
+        mutex_lock(&devices_lock);
+
+        list_for_each_entry(host1x, &devices, list) {
+                err = host1x_register_client(host1x, client);
+                if (!err) {
+                        mutex_unlock(&devices_lock);
+                        return 0;
+                }
+        }
+
+        mutex_unlock(&devices_lock);
+
+        mutex_lock(&clients_lock);
+        list_add_tail(&client->list, &clients);
+        mutex_unlock(&clients_lock);
+
+        return 0;
+}
+EXPORT_SYMBOL(host1x_client_register);
+
+int host1x_client_unregister(struct host1x_client *client)
+{
+        struct host1x_client *c;
+        struct host1x *host1x;
+        int err;
+
+        mutex_lock(&devices_lock);
+
+        list_for_each_entry(host1x, &devices, list) {
+                err = host1x_unregister_client(host1x, client);
+                if (!err) {
+                        mutex_unlock(&devices_lock);
+                        return 0;
+                }
+        }
+
+        mutex_unlock(&devices_lock);
+        mutex_lock(&clients_lock);
+
+        list_for_each_entry(c, &clients, list) {
+                if (c == client) {
+                        list_del_init(&c->list);
+                        break;
+                }
+        }
+
+        mutex_unlock(&clients_lock);
+
+        return 0;
+}
+EXPORT_SYMBOL(host1x_client_unregister);
diff --git a/drivers/gpu/host1x/host1x_client.h b/drivers/gpu/host1x/bus.h
index 9b85f10f4a44..4099e99212c8 100644
--- a/drivers/gpu/host1x/host1x_client.h
+++ b/drivers/gpu/host1x/bus.h
@@ -1,5 +1,6 @@
 /*
- * Copyright (c) 2013, NVIDIA Corporation.
+ * Copyright (C) 2012 Avionic Design GmbH
+ * Copyright (C) 2012-2013, NVIDIA Corporation
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms and conditions of the GNU General Public License,
@@ -14,22 +15,15 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef HOST1X_CLIENT_H
-#define HOST1X_CLIENT_H
+#ifndef HOST1X_BUS_H
+#define HOST1X_BUS_H
 
-struct device;
-struct platform_device;
+struct host1x;
 
-#ifdef CONFIG_DRM_TEGRA
-int host1x_drm_alloc(struct platform_device *pdev);
-#else
-static inline int host1x_drm_alloc(struct platform_device *pdev)
-{
-        return 0;
-}
-#endif
+int host1x_bus_init(void);
+void host1x_bus_exit(void);
 
-void host1x_set_drm_data(struct device *dev, void *data);
-void *host1x_get_drm_data(struct device *dev);
+int host1x_register(struct host1x *host1x);
+int host1x_unregister(struct host1x *host1x);
 
 #endif
diff --git a/drivers/gpu/host1x/cdma.c b/drivers/gpu/host1x/cdma.c
index de72172d3b5f..3995255b16c7 100644
--- a/drivers/gpu/host1x/cdma.c
+++ b/drivers/gpu/host1x/cdma.c
@@ -20,6 +20,7 @@
 #include <asm/cacheflush.h>
 #include <linux/device.h>
 #include <linux/dma-mapping.h>
+#include <linux/host1x.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <linux/kfifo.h>
@@ -30,7 +31,6 @@
 #include "channel.h"
 #include "dev.h"
 #include "debug.h"
-#include "host1x_bo.h"
 #include "job.h"
 
 /*
diff --git a/drivers/gpu/host1x/channel.h b/drivers/gpu/host1x/channel.h
index 48723b8eea42..df767cf90d51 100644
--- a/drivers/gpu/host1x/channel.h
+++ b/drivers/gpu/host1x/channel.h
@@ -40,12 +40,6 @@ struct host1x_channel {
 /* channel list operations */
 int host1x_channel_list_init(struct host1x *host);
 
-struct host1x_channel *host1x_channel_request(struct device *dev);
-void host1x_channel_free(struct host1x_channel *channel);
-struct host1x_channel *host1x_channel_get(struct host1x_channel *channel);
-void host1x_channel_put(struct host1x_channel *channel);
-int host1x_job_submit(struct host1x_job *job);
-
 #define host1x_for_each_channel(host, channel)                          \
         list_for_each_entry(channel, &host->chlist.list, list)
 
diff --git a/drivers/gpu/host1x/dev.c b/drivers/gpu/host1x/dev.c
index 471630299878..80da003d63de 100644
--- a/drivers/gpu/host1x/dev.c
+++ b/drivers/gpu/host1x/dev.c
@@ -27,24 +27,13 @@
 #define CREATE_TRACE_POINTS
 #include <trace/events/host1x.h>
 
+#include "bus.h"
 #include "dev.h"
 #include "intr.h"
 #include "channel.h"
 #include "debug.h"
 #include "hw/host1x01.h"
-#include "host1x_client.h"
-
-void host1x_set_drm_data(struct device *dev, void *data)
-{
-        struct host1x *host1x = dev_get_drvdata(dev);
-        host1x->drm_data = data;
-}
-
-void *host1x_get_drm_data(struct device *dev)
-{
-        struct host1x *host1x = dev_get_drvdata(dev);
-        return host1x ? host1x->drm_data : NULL;
-}
+#include "hw/host1x02.h"
 
 void host1x_sync_writel(struct host1x *host1x, u32 v, u32 r)
 {
@@ -79,7 +68,17 @@ static const struct host1x_info host1x01_info = {
         .sync_offset    = 0x3000,
 };
 
+static const struct host1x_info host1x02_info = {
+        .nb_channels = 9,
+        .nb_pts = 32,
+        .nb_mlocks = 16,
+        .nb_bases = 12,
+        .init = host1x02_init,
+        .sync_offset = 0x3000,
+};
+
 static struct of_device_id host1x_of_match[] = {
+        { .compatible = "nvidia,tegra114-host1x", .data = &host1x02_info, },
         { .compatible = "nvidia,tegra30-host1x", .data = &host1x01_info, },
         { .compatible = "nvidia,tegra20-host1x", .data = &host1x01_info, },
         { },
@@ -114,6 +113,9 @@ static int host1x_probe(struct platform_device *pdev)
         if (!host)
                 return -ENOMEM;
 
+        mutex_init(&host->devices_lock);
+        INIT_LIST_HEAD(&host->devices);
+        INIT_LIST_HEAD(&host->list);
         host->dev = &pdev->dev;
         host->info = id->data;
 
@@ -152,7 +154,7 @@ static int host1x_probe(struct platform_device *pdev)
         err = host1x_syncpt_init(host);
         if (err) {
                 dev_err(&pdev->dev, "failed to initialize syncpts\n");
-                return err;
+                goto fail_unprepare_disable;
         }
 
         err = host1x_intr_init(host, syncpt_irq);
@@ -163,19 +165,26 @@ static int host1x_probe(struct platform_device *pdev)
 
         host1x_debug_init(host);
 
-        host1x_drm_alloc(pdev);
+        err = host1x_register(host);
+        if (err < 0)
+                goto fail_deinit_intr;
 
         return 0;
 
+fail_deinit_intr:
+        host1x_intr_deinit(host);
 fail_deinit_syncpt:
         host1x_syncpt_deinit(host);
+fail_unprepare_disable:
+        clk_disable_unprepare(host->clk);
         return err;
 }
 
-static int __exit host1x_remove(struct platform_device *pdev)
+static int host1x_remove(struct platform_device *pdev)
 {
         struct host1x *host = platform_get_drvdata(pdev);
 
+        host1x_unregister(host);
         host1x_intr_deinit(host);
         host1x_syncpt_deinit(host);
         clk_disable_unprepare(host->clk);
@@ -184,59 +193,36 @@ static int __exit host1x_remove(struct platform_device *pdev)
 }
 
 static struct platform_driver tegra_host1x_driver = {
-        .probe = host1x_probe,
-        .remove = __exit_p(host1x_remove),
         .driver = {
-                .owner = THIS_MODULE,
                 .name = "tegra-host1x",
                 .of_match_table = host1x_of_match,
         },
+        .probe = host1x_probe,
+        .remove = host1x_remove,
 };
 
 static int __init tegra_host1x_init(void)
 {
         int err;
 
-        err = platform_driver_register(&tegra_host1x_driver);
+        err = host1x_bus_init();
         if (err < 0)
                 return err;
 
-#ifdef CONFIG_DRM_TEGRA
-        err = platform_driver_register(&tegra_dc_driver);
-        if (err < 0)
-                goto unregister_host1x;
-
-        err = platform_driver_register(&tegra_hdmi_driver);
-        if (err < 0)
-                goto unregister_dc;
-
-        err = platform_driver_register(&tegra_gr2d_driver);
-        if (err < 0)
-                goto unregister_hdmi;
-#endif
+        err = platform_driver_register(&tegra_host1x_driver);
+        if (err < 0) {
+                host1x_bus_exit();
+                return err;
+        }
 
         return 0;
-
-#ifdef CONFIG_DRM_TEGRA
-unregister_hdmi:
-        platform_driver_unregister(&tegra_hdmi_driver);
-unregister_dc:
-        platform_driver_unregister(&tegra_dc_driver);
-unregister_host1x:
-        platform_driver_unregister(&tegra_host1x_driver);
-        return err;
-#endif
 }
 module_init(tegra_host1x_init);
 
 static void __exit tegra_host1x_exit(void)
 {
-#ifdef CONFIG_DRM_TEGRA
-        platform_driver_unregister(&tegra_gr2d_driver);
-        platform_driver_unregister(&tegra_hdmi_driver);
-        platform_driver_unregister(&tegra_dc_driver);
-#endif
         platform_driver_unregister(&tegra_host1x_driver);
+        host1x_bus_exit();
 }
 module_exit(tegra_host1x_exit);
 
diff --git a/drivers/gpu/host1x/dev.h b/drivers/gpu/host1x/dev.h
index bed90a8131be..a61a976e7a42 100644
--- a/drivers/gpu/host1x/dev.h
+++ b/drivers/gpu/host1x/dev.h
@@ -27,6 +27,7 @@
 #include "job.h"
 
 struct host1x_syncpt;
+struct host1x_syncpt_base;
 struct host1x_channel;
 struct host1x_cdma;
 struct host1x_job;
@@ -102,6 +103,7 @@ struct host1x {
 
         void __iomem *regs;
         struct host1x_syncpt *syncpt;
+        struct host1x_syncpt_base *bases;
         struct device *dev;
         struct clk *clk;
 
@@ -125,7 +127,10 @@ struct host1x {
 
         struct dentry *debugfs;
 
-        void *drm_data;
+        struct mutex devices_lock;
+        struct list_head devices;
+
+        struct list_head list;
 };
 
 void host1x_sync_writel(struct host1x *host1x, u32 r, u32 v);
@@ -301,8 +306,4 @@ static inline void host1x_hw_show_mlocks(struct host1x *host, struct output *o)
         host->debug_op->show_mlocks(host, o);
 }
 
-extern struct platform_driver tegra_dc_driver;
-extern struct platform_driver tegra_hdmi_driver;
-extern struct platform_driver tegra_gr2d_driver;
-
 #endif
diff --git a/drivers/gpu/host1x/drm/gr2d.c b/drivers/gpu/host1x/drm/gr2d.c
deleted file mode 100644
index 27ffcf15a4b4..000000000000
--- a/drivers/gpu/host1x/drm/gr2d.c
+++ /dev/null
@@ -1,343 +0,0 @@
-/*
- * drivers/video/tegra/host/gr2d/gr2d.c
- *
- * Tegra Graphics 2D
- *
- * Copyright (c) 2012-2013, NVIDIA Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#include <linux/export.h>
-#include <linux/of.h>
-#include <linux/of_device.h>
-#include <linux/clk.h>
-
-#include "channel.h"
-#include "drm.h"
-#include "gem.h"
-#include "job.h"
-#include "host1x.h"
-#include "host1x_bo.h"
-#include "host1x_client.h"
-#include "syncpt.h"
-
-struct gr2d {
-        struct host1x_client client;
-        struct clk *clk;
-        struct host1x_channel *channel;
-        unsigned long *addr_regs;
-};
-
-static inline struct gr2d *to_gr2d(struct host1x_client *client)
-{
-        return container_of(client, struct gr2d, client);
-}
-
-static int gr2d_is_addr_reg(struct device *dev, u32 class, u32 reg);
-
-static int gr2d_client_init(struct host1x_client *client,
-                            struct drm_device *drm)
-{
-        return 0;
-}
-
-static int gr2d_client_exit(struct host1x_client *client)
-{
-        return 0;
-}
-
-static int gr2d_open_channel(struct host1x_client *client,
-                             struct host1x_drm_context *context)
-{
-        struct gr2d *gr2d = to_gr2d(client);
-
-        context->channel = host1x_channel_get(gr2d->channel);
-
-        if (!context->channel)
-                return -ENOMEM;
-
-        return 0;
-}
-
-static void gr2d_close_channel(struct host1x_drm_context *context)
-{
-        host1x_channel_put(context->channel);
-}
-
-static struct host1x_bo *host1x_bo_lookup(struct drm_device *drm,
-                                          struct drm_file *file,
-                                          u32 handle)
-{
-        struct drm_gem_object *gem;
-        struct tegra_bo *bo;
-
-        gem = drm_gem_object_lookup(drm, file, handle);
-        if (!gem)
-                return NULL;
-
-        mutex_lock(&drm->struct_mutex);
-        drm_gem_object_unreference(gem);
-        mutex_unlock(&drm->struct_mutex);
-
-        bo = to_tegra_bo(gem);
-        return &bo->base;
-}
-
-static int gr2d_submit(struct host1x_drm_context *context,
-                       struct drm_tegra_submit *args, struct drm_device *drm,
-                       struct drm_file *file)
-{
-        struct host1x_job *job;
-        unsigned int num_cmdbufs = args->num_cmdbufs;
-        unsigned int num_relocs = args->num_relocs;
-        unsigned int num_waitchks = args->num_waitchks;
-        struct drm_tegra_cmdbuf __user *cmdbufs =
-                (void * __user)(uintptr_t)args->cmdbufs;
-        struct drm_tegra_reloc __user *relocs =
-                (void * __user)(uintptr_t)args->relocs;
-        struct drm_tegra_waitchk __user *waitchks =
-                (void * __user)(uintptr_t)args->waitchks;
-        struct drm_tegra_syncpt syncpt;
-        int err;
-
-        /* We don't yet support other than one syncpt_incr struct per submit */
-        if (args->num_syncpts != 1)
-                return -EINVAL;
-
-        job = host1x_job_alloc(context->channel, args->num_cmdbufs,
-                               args->num_relocs, args->num_waitchks);
-        if (!job)
-                return -ENOMEM;
-
-        job->num_relocs = args->num_relocs;
-        job->num_waitchk = args->num_waitchks;
-        job->client = (u32)args->context;
-        job->class = context->client->class;
-        job->serialize = true;
-
-        while (num_cmdbufs) {
-                struct drm_tegra_cmdbuf cmdbuf;
-                struct host1x_bo *bo;
-
-                err = copy_from_user(&cmdbuf, cmdbufs, sizeof(cmdbuf));
-                if (err)
-                        goto fail;
-
-                bo = host1x_bo_lookup(drm, file, cmdbuf.handle);
-                if (!bo) {
-                        err = -ENOENT;
-                        goto fail;
-                }
-
-                host1x_job_add_gather(job, bo, cmdbuf.words, cmdbuf.offset);
-                num_cmdbufs--;
-                cmdbufs++;
-        }
-
-        err = copy_from_user(job->relocarray, relocs,
-                             sizeof(*relocs) * num_relocs);
-        if (err)
-                goto fail;
-
-        while (num_relocs--) {
-                struct host1x_reloc *reloc = &job->relocarray[num_relocs];
-                struct host1x_bo *cmdbuf, *target;
-
-                cmdbuf = host1x_bo_lookup(drm, file, (u32)reloc->cmdbuf);
-                target = host1x_bo_lookup(drm, file, (u32)reloc->target);
-
-                reloc->cmdbuf = cmdbuf;
-                reloc->target = target;
-
-                if (!reloc->target || !reloc->cmdbuf) {
-                        err = -ENOENT;
-                        goto fail;
-                }
-        }
-
-        err = copy_from_user(job->waitchk, waitchks,
-                             sizeof(*waitchks) * num_waitchks);
-        if (err)
-                goto fail;
-
-        err = copy_from_user(&syncpt, (void * __user)(uintptr_t)args->syncpts,
-                             sizeof(syncpt));
-        if (err)
-                goto fail;
-
-        job->syncpt_id = syncpt.id;
-        job->syncpt_incrs = syncpt.incrs;
-        job->timeout = 10000;
-        job->is_addr_reg = gr2d_is_addr_reg;
-
-        if (args->timeout && args->timeout < 10000)
-                job->timeout = args->timeout;
-
-        err = host1x_job_pin(job, context->client->dev);
-        if (err)
-                goto fail;
-
-        err = host1x_job_submit(job);
-        if (err)
-                goto fail_submit;
-
-        args->fence = job->syncpt_end;
-
-        host1x_job_put(job);
-        return 0;
-
-fail_submit:
-        host1x_job_unpin(job);
-fail:
-        host1x_job_put(job);
-        return err;
-}
-
-static struct host1x_client_ops gr2d_client_ops = {
-        .drm_init = gr2d_client_init,
-        .drm_exit = gr2d_client_exit,
-        .open_channel = gr2d_open_channel,
-        .close_channel = gr2d_close_channel,
-        .submit = gr2d_submit,
-};
-
-static void gr2d_init_addr_reg_map(struct device *dev, struct gr2d *gr2d)
-{
-        const u32 gr2d_addr_regs[] = {0x1a, 0x1b, 0x26, 0x2b, 0x2c, 0x2d, 0x31,
-                                      0x32, 0x48, 0x49, 0x4a, 0x4b, 0x4c};
-        unsigned long *bitmap;
-        int i;
-
-        bitmap = devm_kzalloc(dev, DIV_ROUND_UP(256, BITS_PER_BYTE),
-                              GFP_KERNEL);
-
-        for (i = 0; i < ARRAY_SIZE(gr2d_addr_regs); ++i) {
-                u32 reg = gr2d_addr_regs[i];
-                bitmap[BIT_WORD(reg)] |= BIT_MASK(reg);
-        }
-
-        gr2d->addr_regs = bitmap;
-}
-
-static int gr2d_is_addr_reg(struct device *dev, u32 class, u32 reg)
-{
-        struct gr2d *gr2d = dev_get_drvdata(dev);
-
-        switch (class) {
-        case HOST1X_CLASS_HOST1X:
-                return reg == 0x2b;
-        case HOST1X_CLASS_GR2D:
-        case HOST1X_CLASS_GR2D_SB:
-                reg &= 0xff;
-                if (gr2d->addr_regs[BIT_WORD(reg)] & BIT_MASK(reg))
-                        return 1;
-        default:
-                return 0;
-        }
-}
-
-static const struct of_device_id gr2d_match[] = {
-        { .compatible = "nvidia,tegra30-gr2d" },
-        { .compatible = "nvidia,tegra20-gr2d" },
-        { },
-};
-
-static int gr2d_probe(struct platform_device *pdev)
-{
-        struct device *dev = &pdev->dev;
-        struct host1x_drm *host1x = host1x_get_drm_data(dev->parent);
-        int err;
-        struct gr2d *gr2d = NULL;
-        struct host1x_syncpt **syncpts;
-
-        gr2d = devm_kzalloc(dev, sizeof(*gr2d), GFP_KERNEL);
-        if (!gr2d)
-                return -ENOMEM;
-
-        syncpts = devm_kzalloc(dev, sizeof(*syncpts), GFP_KERNEL);
-        if (!syncpts)
-                return -ENOMEM;
-
-        gr2d->clk = devm_clk_get(dev, NULL);
-        if (IS_ERR(gr2d->clk)) {
-                dev_err(dev, "cannot get clock\n");
-                return PTR_ERR(gr2d->clk);
-        }
-
-        err = clk_prepare_enable(gr2d->clk);
-        if (err) {
-                dev_err(dev, "cannot turn on clock\n");
-                return err;
-        }
-
-        gr2d->channel = host1x_channel_request(dev);
-        if (!gr2d->channel)
-                return -ENOMEM;
-
-        *syncpts = host1x_syncpt_request(dev, false);
-        if (!(*syncpts)) {
-                host1x_channel_free(gr2d->channel);
-                return -ENOMEM;
-        }
-
-        gr2d->client.ops = &gr2d_client_ops;
-        gr2d->client.dev = dev;
-        gr2d->client.class = HOST1X_CLASS_GR2D;
-        gr2d->client.syncpts = syncpts;
-        gr2d->client.num_syncpts = 1;
-
-        err = host1x_register_client(host1x, &gr2d->client);
-        if (err < 0) {
-                dev_err(dev, "failed to register host1x client: %d\n", err);
-                return err;
-        }
-
-        gr2d_init_addr_reg_map(dev, gr2d);
-
-        platform_set_drvdata(pdev, gr2d);
-
-        return 0;
-}
-
-static int __exit gr2d_remove(struct platform_device *pdev)
-{
-        struct host1x_drm *host1x = host1x_get_drm_data(pdev->dev.parent);
-        struct gr2d *gr2d = platform_get_drvdata(pdev);
-        unsigned int i;
-        int err;
-
-        err = host1x_unregister_client(host1x, &gr2d->client);
-        if (err < 0) {
-                dev_err(&pdev->dev, "failed to unregister client: %d\n", err);
-                return err;
-        }
-
-        for (i = 0; i < gr2d->client.num_syncpts; i++)
-                host1x_syncpt_free(gr2d->client.syncpts[i]);
-
-        host1x_channel_free(gr2d->channel);
-        clk_disable_unprepare(gr2d->clk);
-
-        return 0;
-}
-
-struct platform_driver tegra_gr2d_driver = {
-        .probe = gr2d_probe,
-        .remove = __exit_p(gr2d_remove),
-        .driver = {
-                .owner = THIS_MODULE,
-                .name = "gr2d",
-                .of_match_table = gr2d_match,
-        }
-};
diff --git a/drivers/gpu/host1x/host1x.h b/drivers/gpu/host1x/host1x.h
deleted file mode 100644
index a2bc1e65e972..000000000000
--- a/drivers/gpu/host1x/host1x.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Tegra host1x driver
- *
- * Copyright (c) 2009-2013, NVIDIA Corporation. All rights reserved.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with this program; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
- */
-
-#ifndef __LINUX_HOST1X_H
-#define __LINUX_HOST1X_H
-
-enum host1x_class {
-        HOST1X_CLASS_HOST1X     = 0x1,
-        HOST1X_CLASS_GR2D       = 0x51,
-        HOST1X_CLASS_GR2D_SB    = 0x52
-};
-
-#endif
diff --git a/drivers/gpu/host1x/host1x_bo.h b/drivers/gpu/host1x/host1x_bo.h
deleted file mode 100644
index 4c1f10bd773d..000000000000
--- a/drivers/gpu/host1x/host1x_bo.h
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Tegra host1x Memory Management Abstraction header
- *
- * Copyright (c) 2012-2013, NVIDIA Corporation.
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef _HOST1X_BO_H
-#define _HOST1X_BO_H
-
-struct host1x_bo;
-
-struct host1x_bo_ops {
-        struct host1x_bo *(*get)(struct host1x_bo *bo);
-        void (*put)(struct host1x_bo *bo);
-        dma_addr_t (*pin)(struct host1x_bo *bo, struct sg_table **sgt);
-        void (*unpin)(struct host1x_bo *bo, struct sg_table *sgt);
-        void *(*mmap)(struct host1x_bo *bo);
-        void (*munmap)(struct host1x_bo *bo, void *addr);
-        void *(*kmap)(struct host1x_bo *bo, unsigned int pagenum);
-        void (*kunmap)(struct host1x_bo *bo, unsigned int pagenum, void *addr);
-};
-
-struct host1x_bo {
-        const struct host1x_bo_ops *ops;
-};
-
-static inline void host1x_bo_init(struct host1x_bo *bo,
-                                  const struct host1x_bo_ops *ops)
-{
-        bo->ops = ops;
-}
-
-static inline struct host1x_bo *host1x_bo_get(struct host1x_bo *bo)
-{
-        return bo->ops->get(bo);
-}
-
-static inline void host1x_bo_put(struct host1x_bo *bo)
-{
-        bo->ops->put(bo);
-}
-
-static inline dma_addr_t host1x_bo_pin(struct host1x_bo *bo,
-                                       struct sg_table **sgt)
-{
-        return bo->ops->pin(bo, sgt);
-}
-
-static inline void host1x_bo_unpin(struct host1x_bo *bo, struct sg_table *sgt)
-{
-        bo->ops->unpin(bo, sgt);
-}
-
-static inline void *host1x_bo_mmap(struct host1x_bo *bo)
-{
-        return bo->ops->mmap(bo);
-}
-
-static inline void host1x_bo_munmap(struct host1x_bo *bo, void *addr)
-{
-        bo->ops->munmap(bo, addr);
-}
-
-static inline void *host1x_bo_kmap(struct host1x_bo *bo, unsigned int pagenum)
-{
-        return bo->ops->kmap(bo, pagenum);
-}
-
-static inline void host1x_bo_kunmap(struct host1x_bo *bo,
-                                    unsigned int pagenum, void *addr)
-{
-        bo->ops->kunmap(bo, pagenum, addr);
-}
-
-#endif
diff --git a/drivers/gpu/host1x/hw/Makefile b/drivers/gpu/host1x/hw/Makefile
deleted file mode 100644
index 9b50863a2236..000000000000
--- a/drivers/gpu/host1x/hw/Makefile
+++ /dev/null
@@ -1,6 +0,0 @@
-ccflags-y = -Idrivers/gpu/host1x
-
-host1x-hw-objs  = \
-        host1x01.o
-
-obj-$(CONFIG_TEGRA_HOST1X) += host1x-hw.o
diff --git a/drivers/gpu/host1x/hw/cdma_hw.c b/drivers/gpu/host1x/hw/cdma_hw.c
index 2ee4ad55c4db..37e2a63241a9 100644
--- a/drivers/gpu/host1x/hw/cdma_hw.c
+++ b/drivers/gpu/host1x/hw/cdma_hw.c
@@ -20,10 +20,10 @@
 #include <linux/scatterlist.h>
 #include <linux/dma-mapping.h>
 
-#include "cdma.h"
-#include "channel.h"
-#include "dev.h"
-#include "debug.h"
+#include "../cdma.h"
+#include "../channel.h"
+#include "../dev.h"
+#include "../debug.h"
 
 /*
  * Put the restart at the end of pushbuffer memor
diff --git a/drivers/gpu/host1x/hw/channel_hw.c b/drivers/gpu/host1x/hw/channel_hw.c
index ee199623e365..4608257ab656 100644
--- a/drivers/gpu/host1x/hw/channel_hw.c
+++ b/drivers/gpu/host1x/hw/channel_hw.c
@@ -16,15 +16,15 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/host1x.h>
 #include <linux/slab.h>
+
 #include <trace/events/host1x.h>
 
-#include "host1x.h"
-#include "host1x_bo.h"
-#include "channel.h"
-#include "dev.h"
-#include "intr.h"
-#include "job.h"
+#include "../channel.h"
+#include "../dev.h"
+#include "../intr.h"
+#include "../job.h"
 
 #define HOST1X_CHANNEL_SIZE 16384
 #define TRACE_MAX_LENGTH 128U
@@ -67,6 +67,22 @@ static void submit_gathers(struct host1x_job *job)
         }
 }
 
+static inline void synchronize_syncpt_base(struct host1x_job *job)
+{
+        struct host1x *host = dev_get_drvdata(job->channel->dev->parent);
+        struct host1x_syncpt *sp = host->syncpt + job->syncpt_id;
+        u32 id, value;
+
+        value = host1x_syncpt_read_max(sp);
+        id = sp->base->id;
+
+        host1x_cdma_push(&job->channel->cdma,
+                         host1x_opcode_setclass(HOST1X_CLASS_HOST1X,
+                                HOST1X_UCLASS_LOAD_SYNCPT_BASE, 1),
+                         HOST1X_UCLASS_LOAD_SYNCPT_BASE_BASE_INDX_F(id) |
+                         HOST1X_UCLASS_LOAD_SYNCPT_BASE_VALUE_F(value));
+}
+
 static int channel_submit(struct host1x_job *job)
 {
         struct host1x_channel *ch = job->channel;
@@ -118,6 +134,10 @@ static int channel_submit(struct host1x_job *job)
                                         host1x_syncpt_read_max(sp)));
         }
 
+        /* Synchronize base register to allow using it for relative waiting */
+        if (sp->base)
+                synchronize_syncpt_base(job);
+
         syncval = host1x_syncpt_incr_max(sp, user_syncpt_incrs);
 
         job->syncpt_end = syncval;
diff --git a/drivers/gpu/host1x/hw/debug_hw.c b/drivers/gpu/host1x/hw/debug_hw.c
index 334c038052f5..640c75ca5a8b 100644
--- a/drivers/gpu/host1x/hw/debug_hw.c
+++ b/drivers/gpu/host1x/hw/debug_hw.c
@@ -15,18 +15,10 @@
  *
  */
 
-#include <linux/debugfs.h>
-#include <linux/seq_file.h>
-#include <linux/mm.h>
-#include <linux/scatterlist.h>
-
-#include <linux/io.h>
-
-#include "dev.h"
-#include "debug.h"
-#include "cdma.h"
-#include "channel.h"
-#include "host1x_bo.h"
+#include "../dev.h"
+#include "../debug.h"
+#include "../cdma.h"
+#include "../channel.h"
 
 #define HOST1X_DEBUG_MAX_PAGE_OFFSET 102400
 
diff --git a/drivers/gpu/host1x/hw/host1x01.c b/drivers/gpu/host1x/hw/host1x01.c
index a14e91cd1e58..859b73beb4d0 100644
--- a/drivers/gpu/host1x/hw/host1x01.c
+++ b/drivers/gpu/host1x/hw/host1x01.c
@@ -17,17 +17,17 @@
  */
 
 /* include hw specification */
-#include "hw/host1x01.h"
-#include "hw/host1x01_hardware.h"
+#include "host1x01.h"
+#include "host1x01_hardware.h"
 
 /* include code */
-#include "hw/cdma_hw.c"
-#include "hw/channel_hw.c"
-#include "hw/debug_hw.c"
-#include "hw/intr_hw.c"
-#include "hw/syncpt_hw.c"
+#include "cdma_hw.c"
+#include "channel_hw.c"
+#include "debug_hw.c"
+#include "intr_hw.c"
+#include "syncpt_hw.c"
 
-#include "dev.h"
+#include "../dev.h"
 
 int host1x01_init(struct host1x *host)
 {
diff --git a/drivers/gpu/host1x/hw/host1x02.c b/drivers/gpu/host1x/hw/host1x02.c
new file mode 100644
index 000000000000..e98caca0ca42
--- /dev/null
+++ b/drivers/gpu/host1x/hw/host1x02.c
@@ -0,0 +1,42 @@
+/*
+ * Host1x init for Tegra114 SoCs
+ *
+ * Copyright (c) 2013 NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+/* include hw specification */
+#include "host1x01.h"
+#include "host1x01_hardware.h"
+
+/* include code */
+#include "cdma_hw.c"
+#include "channel_hw.c"
+#include "debug_hw.c"
+#include "intr_hw.c"
+#include "syncpt_hw.c"
+
+#include "../dev.h"
+
+int host1x02_init(struct host1x *host)
+{
+        host->channel_op = &host1x_channel_ops;
+        host->cdma_op = &host1x_cdma_ops;
+        host->cdma_pb_op = &host1x_pushbuffer_ops;
+        host->syncpt_op = &host1x_syncpt_ops;
+        host->intr_op = &host1x_intr_ops;
+        host->debug_op = &host1x_debug_ops;
+
+        return 0;
+}
diff --git a/drivers/gpu/host1x/hw/host1x02.h b/drivers/gpu/host1x/hw/host1x02.h
new file mode 100644
index 000000000000..f7486609a90e
--- /dev/null
+++ b/drivers/gpu/host1x/hw/host1x02.h
@@ -0,0 +1,26 @@
+/*
+ * Host1x init for Tegra114 SoCs
+ *
+ * Copyright (c) 2013 NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef HOST1X_HOST1X02_H
+#define HOST1X_HOST1X02_H
+
+struct host1x;
+
+int host1x02_init(struct host1x *host);
+
+#endif
diff --git a/drivers/gpu/host1x/hw/hw_host1x01_uclass.h b/drivers/gpu/host1x/hw/hw_host1x01_uclass.h
index 42f3ce19ca32..f7553599ee27 100644
--- a/drivers/gpu/host1x/hw/hw_host1x01_uclass.h
+++ b/drivers/gpu/host1x/hw/hw_host1x01_uclass.h
@@ -111,6 +111,12 @@ static inline u32 host1x_uclass_wait_syncpt_base_offset_f(u32 v)
 }
 #define HOST1X_UCLASS_WAIT_SYNCPT_BASE_OFFSET_F(v) \
         host1x_uclass_wait_syncpt_base_offset_f(v)
+static inline u32 host1x_uclass_load_syncpt_base_r(void)
+{
+        return 0xb;
+}
+#define HOST1X_UCLASS_LOAD_SYNCPT_BASE \
+        host1x_uclass_load_syncpt_base_r()
 static inline u32 host1x_uclass_load_syncpt_base_base_indx_f(u32 v)
 {
         return (v & 0xff) << 24;
diff --git a/drivers/gpu/host1x/hw/hw_host1x02_channel.h b/drivers/gpu/host1x/hw/hw_host1x02_channel.h
new file mode 100644
index 000000000000..e490bcde33fe
--- /dev/null
+++ b/drivers/gpu/host1x/hw/hw_host1x02_channel.h
@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2013 NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+ /*
+  * Function naming determines intended use:
+  *
+  *     <x>_r(void) : Returns the offset for register <x>.
+  *
+  *     <x>_w(void) : Returns the word offset for word (4 byte) element <x>.
+  *
+  *     <x>_<y>_s(void) : Returns size of field <y> of register <x> in bits.
+  *
+  *     <x>_<y>_f(u32 v) : Returns a value based on 'v' which has been shifted
+  *         and masked to place it at field <y> of register <x>.  This value
+  *         can be |'d with others to produce a full register value for
+  *         register <x>.
+  *
+  *     <x>_<y>_m(void) : Returns a mask for field <y> of register <x>.  This
+  *         value can be ~'d and then &'d to clear the value of field <y> for
+  *         register <x>.
+  *
+  *     <x>_<y>_<z>_f(void) : Returns the constant value <z> after being shifted
+  *         to place it at field <y> of register <x>.  This value can be |'d
+  *         with others to produce a full register value for <x>.
+  *
+  *     <x>_<y>_v(u32 r) : Returns the value of field <y> from a full register
+  *         <x> value 'r' after being shifted to place its LSB at bit 0.
+  *         This value is suitable for direct comparison with other unshifted
+  *         values appropriate for use in field <y> of register <x>.
+  *
+  *     <x>_<y>_<z>_v(void) : Returns the constant value for <z> defined for
+  *         field <y> of register <x>.  This value is suitable for direct
+  *         comparison with unshifted values appropriate for use in field <y>
+  *         of register <x>.
+  */
+
+#ifndef HOST1X_HW_HOST1X02_CHANNEL_H
+#define HOST1X_HW_HOST1X02_CHANNEL_H
+
+static inline u32 host1x_channel_fifostat_r(void)
+{
+        return 0x0;
+}
+#define HOST1X_CHANNEL_FIFOSTAT \
+        host1x_channel_fifostat_r()
+static inline u32 host1x_channel_fifostat_cfempty_v(u32 r)
+{
+        return (r >> 11) & 0x1;
+}
+#define HOST1X_CHANNEL_FIFOSTAT_CFEMPTY_V(r) \
+        host1x_channel_fifostat_cfempty_v(r)
+static inline u32 host1x_channel_dmastart_r(void)
+{
+        return 0x14;
+}
+#define HOST1X_CHANNEL_DMASTART \
+        host1x_channel_dmastart_r()
+static inline u32 host1x_channel_dmaput_r(void)
+{
+        return 0x18;
+}
+#define HOST1X_CHANNEL_DMAPUT \
+        host1x_channel_dmaput_r()
+static inline u32 host1x_channel_dmaget_r(void)
+{
+        return 0x1c;
+}
+#define HOST1X_CHANNEL_DMAGET \
+        host1x_channel_dmaget_r()
+static inline u32 host1x_channel_dmaend_r(void)
+{
+        return 0x20;
+}
+#define HOST1X_CHANNEL_DMAEND \
+        host1x_channel_dmaend_r()
+static inline u32 host1x_channel_dmactrl_r(void)
+{
+        return 0x24;
+}
+#define HOST1X_CHANNEL_DMACTRL \
+        host1x_channel_dmactrl_r()
+static inline u32 host1x_channel_dmactrl_dmastop(void)
+{
+        return 1 << 0;
+}
+#define HOST1X_CHANNEL_DMACTRL_DMASTOP \
+        host1x_channel_dmactrl_dmastop()
+static inline u32 host1x_channel_dmactrl_dmastop_v(u32 r)
+{
+        return (r >> 0) & 0x1;
+}
+#define HOST1X_CHANNEL_DMACTRL_DMASTOP_V(r) \
+        host1x_channel_dmactrl_dmastop_v(r)
+static inline u32 host1x_channel_dmactrl_dmagetrst(void)
+{
+        return 1 << 1;
+}
+#define HOST1X_CHANNEL_DMACTRL_DMAGETRST \
+        host1x_channel_dmactrl_dmagetrst()
+static inline u32 host1x_channel_dmactrl_dmainitget(void)
+{
+        return 1 << 2;
+}
+#define HOST1X_CHANNEL_DMACTRL_DMAINITGET \
+        host1x_channel_dmactrl_dmainitget()
+
+#endif
diff --git a/drivers/gpu/host1x/hw/hw_host1x02_sync.h b/drivers/gpu/host1x/hw/hw_host1x02_sync.h
new file mode 100644
index 000000000000..4495401525e8
--- /dev/null
+++ b/drivers/gpu/host1x/hw/hw_host1x02_sync.h
@@ -0,0 +1,243 @@
+/*
+ * Copyright (c) 2013 NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+ /*
+  * Function naming determines intended use:
+  *
+  *     <x>_r(void) : Returns the offset for register <x>.
+  *
+  *     <x>_w(void) : Returns the word offset for word (4 byte) element <x>.
+  *
+  *     <x>_<y>_s(void) : Returns size of field <y> of register <x> in bits.
+  *
+  *     <x>_<y>_f(u32 v) : Returns a value based on 'v' which has been shifted
+  *         and masked to place it at field <y> of register <x>.  This value
+  *         can be |'d with others to produce a full register value for
+  *         register <x>.
+  *
+  *     <x>_<y>_m(void) : Returns a mask for field <y> of register <x>.  This
+  *         value can be ~'d and then &'d to clear the value of field <y> for
+  *         register <x>.
+  *
+  *     <x>_<y>_<z>_f(void) : Returns the constant value <z> after being shifted
+  *         to place it at field <y> of register <x>.  This value can be |'d
+  *         with others to produce a full register value for <x>.
+  *
+  *     <x>_<y>_v(u32 r) : Returns the value of field <y> from a full register
+  *         <x> value 'r' after being shifted to place its LSB at bit 0.
+  *         This value is suitable for direct comparison with other unshifted
+  *         values appropriate for use in field <y> of register <x>.
+  *
+  *     <x>_<y>_<z>_v(void) : Returns the constant value for <z> defined for
+  *         field <y> of register <x>.  This value is suitable for direct
+  *         comparison with unshifted values appropriate for use in field <y>
+  *         of register <x>.
+  */
+
+#ifndef HOST1X_HW_HOST1X02_SYNC_H
+#define HOST1X_HW_HOST1X02_SYNC_H
+
+#define REGISTER_STRIDE 4
+
+static inline u32 host1x_sync_syncpt_r(unsigned int id)
+{
+        return 0x400 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_SYNCPT(id) \
+        host1x_sync_syncpt_r(id)
+static inline u32 host1x_sync_syncpt_thresh_cpu0_int_status_r(unsigned int id)
+{
+        return 0x40 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS(id) \
+        host1x_sync_syncpt_thresh_cpu0_int_status_r(id)
+static inline u32 host1x_sync_syncpt_thresh_int_disable_r(unsigned int id)
+{
+        return 0x60 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_SYNCPT_THRESH_INT_DISABLE(id) \
+        host1x_sync_syncpt_thresh_int_disable_r(id)
+static inline u32 host1x_sync_syncpt_thresh_int_enable_cpu0_r(unsigned int id)
+{
+        return 0x68 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_SYNCPT_THRESH_INT_ENABLE_CPU0(id) \
+        host1x_sync_syncpt_thresh_int_enable_cpu0_r(id)
+static inline u32 host1x_sync_cf_setup_r(unsigned int channel)
+{
+        return 0x80 + channel * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_CF_SETUP(channel) \
+        host1x_sync_cf_setup_r(channel)
+static inline u32 host1x_sync_cf_setup_base_v(u32 r)
+{
+        return (r >> 0) & 0x3ff;
+}
+#define HOST1X_SYNC_CF_SETUP_BASE_V(r) \
+        host1x_sync_cf_setup_base_v(r)
+static inline u32 host1x_sync_cf_setup_limit_v(u32 r)
+{
+        return (r >> 16) & 0x3ff;
+}
+#define HOST1X_SYNC_CF_SETUP_LIMIT_V(r) \
+        host1x_sync_cf_setup_limit_v(r)
+static inline u32 host1x_sync_cmdproc_stop_r(void)
+{
+        return 0xac;
+}
+#define HOST1X_SYNC_CMDPROC_STOP \
+        host1x_sync_cmdproc_stop_r()
+static inline u32 host1x_sync_ch_teardown_r(void)
+{
+        return 0xb0;
+}
+#define HOST1X_SYNC_CH_TEARDOWN \
+        host1x_sync_ch_teardown_r()
+static inline u32 host1x_sync_usec_clk_r(void)
+{
+        return 0x1a4;
+}
+#define HOST1X_SYNC_USEC_CLK \
+        host1x_sync_usec_clk_r()
+static inline u32 host1x_sync_ctxsw_timeout_cfg_r(void)
+{
+        return 0x1a8;
+}
+#define HOST1X_SYNC_CTXSW_TIMEOUT_CFG \
+        host1x_sync_ctxsw_timeout_cfg_r()
+static inline u32 host1x_sync_ip_busy_timeout_r(void)
+{
+        return 0x1bc;
+}
+#define HOST1X_SYNC_IP_BUSY_TIMEOUT \
+        host1x_sync_ip_busy_timeout_r()
+static inline u32 host1x_sync_mlock_owner_r(unsigned int id)
+{
+        return 0x340 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_MLOCK_OWNER(id) \
+        host1x_sync_mlock_owner_r(id)
+static inline u32 host1x_sync_mlock_owner_chid_f(u32 v)
+{
+        return (v & 0xf) << 8;
+}
+#define HOST1X_SYNC_MLOCK_OWNER_CHID_F(v) \
+        host1x_sync_mlock_owner_chid_f(v)
+static inline u32 host1x_sync_mlock_owner_cpu_owns_v(u32 r)
+{
+        return (r >> 1) & 0x1;
+}
+#define HOST1X_SYNC_MLOCK_OWNER_CPU_OWNS_V(r) \
+        host1x_sync_mlock_owner_cpu_owns_v(r)
+static inline u32 host1x_sync_mlock_owner_ch_owns_v(u32 r)
+{
+        return (r >> 0) & 0x1;
+}
+#define HOST1X_SYNC_MLOCK_OWNER_CH_OWNS_V(r) \
+        host1x_sync_mlock_owner_ch_owns_v(r)
+static inline u32 host1x_sync_syncpt_int_thresh_r(unsigned int id)
+{
+        return 0x500 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_SYNCPT_INT_THRESH(id) \
+        host1x_sync_syncpt_int_thresh_r(id)
+static inline u32 host1x_sync_syncpt_base_r(unsigned int id)
+{
+        return 0x600 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_SYNCPT_BASE(id) \
+        host1x_sync_syncpt_base_r(id)
+static inline u32 host1x_sync_syncpt_cpu_incr_r(unsigned int id)
+{
+        return 0x700 + id * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_SYNCPT_CPU_INCR(id) \
+        host1x_sync_syncpt_cpu_incr_r(id)
+static inline u32 host1x_sync_cbread_r(unsigned int channel)
+{
+        return 0x720 + channel * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_CBREAD(channel) \
+        host1x_sync_cbread_r(channel)
+static inline u32 host1x_sync_cfpeek_ctrl_r(void)
+{
+        return 0x74c;
+}
+#define HOST1X_SYNC_CFPEEK_CTRL \
+        host1x_sync_cfpeek_ctrl_r()
+static inline u32 host1x_sync_cfpeek_ctrl_addr_f(u32 v)
+{
+        return (v & 0x3ff) << 0;
+}
+#define HOST1X_SYNC_CFPEEK_CTRL_ADDR_F(v) \
+        host1x_sync_cfpeek_ctrl_addr_f(v)
+static inline u32 host1x_sync_cfpeek_ctrl_channr_f(u32 v)
+{
+        return (v & 0xf) << 16;
+}
+#define HOST1X_SYNC_CFPEEK_CTRL_CHANNR_F(v) \
+        host1x_sync_cfpeek_ctrl_channr_f(v)
+static inline u32 host1x_sync_cfpeek_ctrl_ena_f(u32 v)
+{
+        return (v & 0x1) << 31;
+}
+#define HOST1X_SYNC_CFPEEK_CTRL_ENA_F(v) \
+        host1x_sync_cfpeek_ctrl_ena_f(v)
+static inline u32 host1x_sync_cfpeek_read_r(void)
+{
+        return 0x750;
+}
+#define HOST1X_SYNC_CFPEEK_READ \
+        host1x_sync_cfpeek_read_r()
+static inline u32 host1x_sync_cfpeek_ptrs_r(void)
+{
+        return 0x754;
+}
+#define HOST1X_SYNC_CFPEEK_PTRS \
+        host1x_sync_cfpeek_ptrs_r()
+static inline u32 host1x_sync_cfpeek_ptrs_cf_rd_ptr_v(u32 r)
+{
+        return (r >> 0) & 0x3ff;
+}
+#define HOST1X_SYNC_CFPEEK_PTRS_CF_RD_PTR_V(r) \
+        host1x_sync_cfpeek_ptrs_cf_rd_ptr_v(r)
+static inline u32 host1x_sync_cfpeek_ptrs_cf_wr_ptr_v(u32 r)
+{
+        return (r >> 16) & 0x3ff;
+}
+#define HOST1X_SYNC_CFPEEK_PTRS_CF_WR_PTR_V(r) \
+        host1x_sync_cfpeek_ptrs_cf_wr_ptr_v(r)
+static inline u32 host1x_sync_cbstat_r(unsigned int channel)
+{
+        return 0x758 + channel * REGISTER_STRIDE;
+}
+#define HOST1X_SYNC_CBSTAT(channel) \
+        host1x_sync_cbstat_r(channel)
+static inline u32 host1x_sync_cbstat_cboffset_v(u32 r)
+{
+        return (r >> 0) & 0xffff;
+}
+#define HOST1X_SYNC_CBSTAT_CBOFFSET_V(r) \
+        host1x_sync_cbstat_cboffset_v(r)
+static inline u32 host1x_sync_cbstat_cbclass_v(u32 r)
+{
+        return (r >> 16) & 0x3ff;
+}
+#define HOST1X_SYNC_CBSTAT_CBCLASS_V(r) \
+        host1x_sync_cbstat_cbclass_v(r)
+
+#endif
diff --git a/drivers/gpu/host1x/hw/hw_host1x02_uclass.h b/drivers/gpu/host1x/hw/hw_host1x02_uclass.h
new file mode 100644
index 000000000000..a3b3c9874413
--- /dev/null
+++ b/drivers/gpu/host1x/hw/hw_host1x02_uclass.h
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 2013 NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+ /*
+  * Function naming determines intended use:
+  *
+  *     <x>_r(void) : Returns the offset for register <x>.
+  *
+  *     <x>_w(void) : Returns the word offset for word (4 byte) element <x>.
+  *
+  *     <x>_<y>_s(void) : Returns size of field <y> of register <x> in bits.
+  *
+  *     <x>_<y>_f(u32 v) : Returns a value based on 'v' which has been shifted
+  *         and masked to place it at field <y> of register <x>.  This value
+  *         can be |'d with others to produce a full register value for
+  *         register <x>.
+  *
+  *     <x>_<y>_m(void) : Returns a mask for field <y> of register <x>.  This
+  *         value can be ~'d and then &'d to clear the value of field <y> for
+  *         register <x>.
+  *
+  *     <x>_<y>_<z>_f(void) : Returns the constant value <z> after being shifted
+  *         to place it at field <y> of register <x>.  This value can be |'d
+  *         with others to produce a full register value for <x>.
+  *
+  *     <x>_<y>_v(u32 r) : Returns the value of field <y> from a full register
+  *         <x> value 'r' after being shifted to place its LSB at bit 0.
+  *         This value is suitable for direct comparison with other unshifted
+  *         values appropriate for use in field <y> of register <x>.
+  *
+  *     <x>_<y>_<z>_v(void) : Returns the constant value for <z> defined for
+  *         field <y> of register <x>.  This value is suitable for direct
+  *         comparison with unshifted values appropriate for use in field <y>
+  *         of register <x>.
+  */
+
+#ifndef HOST1X_HW_HOST1X02_UCLASS_H
+#define HOST1X_HW_HOST1X02_UCLASS_H
+
+static inline u32 host1x_uclass_incr_syncpt_r(void)
+{
+        return 0x0;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT \
+        host1x_uclass_incr_syncpt_r()
+static inline u32 host1x_uclass_incr_syncpt_cond_f(u32 v)
+{
+        return (v & 0xff) << 8;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT_COND_F(v) \
+        host1x_uclass_incr_syncpt_cond_f(v)
+static inline u32 host1x_uclass_incr_syncpt_indx_f(u32 v)
+{
+        return (v & 0xff) << 0;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT_INDX_F(v) \
+        host1x_uclass_incr_syncpt_indx_f(v)
+static inline u32 host1x_uclass_wait_syncpt_r(void)
+{
+        return 0x8;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT \
+        host1x_uclass_wait_syncpt_r()
+static inline u32 host1x_uclass_wait_syncpt_indx_f(u32 v)
+{
+        return (v & 0xff) << 24;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_INDX_F(v) \
+        host1x_uclass_wait_syncpt_indx_f(v)
+static inline u32 host1x_uclass_wait_syncpt_thresh_f(u32 v)
+{
+        return (v & 0xffffff) << 0;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_THRESH_F(v) \
+        host1x_uclass_wait_syncpt_thresh_f(v)
+static inline u32 host1x_uclass_wait_syncpt_base_r(void)
+{
+        return 0x9;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_BASE \
+        host1x_uclass_wait_syncpt_base_r()
+static inline u32 host1x_uclass_wait_syncpt_base_indx_f(u32 v)
+{
+        return (v & 0xff) << 24;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_BASE_INDX_F(v) \
+        host1x_uclass_wait_syncpt_base_indx_f(v)
+static inline u32 host1x_uclass_wait_syncpt_base_base_indx_f(u32 v)
+{
+        return (v & 0xff) << 16;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_BASE_BASE_INDX_F(v) \
+        host1x_uclass_wait_syncpt_base_base_indx_f(v)
+static inline u32 host1x_uclass_wait_syncpt_base_offset_f(u32 v)
+{
+        return (v & 0xffff) << 0;
+}
+#define HOST1X_UCLASS_WAIT_SYNCPT_BASE_OFFSET_F(v) \
+        host1x_uclass_wait_syncpt_base_offset_f(v)
+static inline u32 host1x_uclass_load_syncpt_base_base_indx_f(u32 v)
+{
+        return (v & 0xff) << 24;
+}
+#define HOST1X_UCLASS_LOAD_SYNCPT_BASE_BASE_INDX_F(v) \
+        host1x_uclass_load_syncpt_base_base_indx_f(v)
+static inline u32 host1x_uclass_load_syncpt_base_value_f(u32 v)
+{
+        return (v & 0xffffff) << 0;
+}
+#define HOST1X_UCLASS_LOAD_SYNCPT_BASE_VALUE_F(v) \
+        host1x_uclass_load_syncpt_base_value_f(v)
+static inline u32 host1x_uclass_incr_syncpt_base_base_indx_f(u32 v)
+{
+        return (v & 0xff) << 24;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT_BASE_BASE_INDX_F(v) \
+        host1x_uclass_incr_syncpt_base_base_indx_f(v)
+static inline u32 host1x_uclass_incr_syncpt_base_offset_f(u32 v)
+{
+        return (v & 0xffffff) << 0;
+}
+#define HOST1X_UCLASS_INCR_SYNCPT_BASE_OFFSET_F(v) \
+        host1x_uclass_incr_syncpt_base_offset_f(v)
+static inline u32 host1x_uclass_indoff_r(void)
+{
+        return 0x2d;
+}
+#define HOST1X_UCLASS_INDOFF \
+        host1x_uclass_indoff_r()
+static inline u32 host1x_uclass_indoff_indbe_f(u32 v)
+{
+        return (v & 0xf) << 28;
+}
+#define HOST1X_UCLASS_INDOFF_INDBE_F(v) \
+        host1x_uclass_indoff_indbe_f(v)
+static inline u32 host1x_uclass_indoff_autoinc_f(u32 v)
+{
+        return (v & 0x1) << 27;
+}
+#define HOST1X_UCLASS_INDOFF_AUTOINC_F(v) \
+        host1x_uclass_indoff_autoinc_f(v)
+static inline u32 host1x_uclass_indoff_indmodid_f(u32 v)
+{
+        return (v & 0xff) << 18;
+}
+#define HOST1X_UCLASS_INDOFF_INDMODID_F(v) \
+        host1x_uclass_indoff_indmodid_f(v)
+static inline u32 host1x_uclass_indoff_indroffset_f(u32 v)
+{
+        return (v & 0xffff) << 2;
+}
+#define HOST1X_UCLASS_INDOFF_INDROFFSET_F(v) \
+        host1x_uclass_indoff_indroffset_f(v)
+static inline u32 host1x_uclass_indoff_rwn_read_v(void)
+{
+        return 1;
+}
+#define HOST1X_UCLASS_INDOFF_INDROFFSET_F(v) \
+        host1x_uclass_indoff_indroffset_f(v)
+
+#endif
diff --git a/drivers/gpu/host1x/hw/intr_hw.c b/drivers/gpu/host1x/hw/intr_hw.c
index b592eef1efcb..b26dcc83bc1b 100644
--- a/drivers/gpu/host1x/hw/intr_hw.c
+++ b/drivers/gpu/host1x/hw/intr_hw.c
@@ -22,8 +22,8 @@
 #include <linux/io.h>
 #include <asm/mach/irq.h>
 
-#include "intr.h"
-#include "dev.h"
+#include "../intr.h"
+#include "../dev.h"
 
 /*
  * Sync point threshold interrupt service function
diff --git a/drivers/gpu/host1x/hw/syncpt_hw.c b/drivers/gpu/host1x/hw/syncpt_hw.c
index 0cf6095d3367..56e85395ac24 100644
--- a/drivers/gpu/host1x/hw/syncpt_hw.c
+++ b/drivers/gpu/host1x/hw/syncpt_hw.c
@@ -18,8 +18,8 @@
 
 #include <linux/io.h>
 
-#include "dev.h"
-#include "syncpt.h"
+#include "../dev.h"
+#include "../syncpt.h"
 
 /*
  * Write the current syncpoint value back to hw.
diff --git a/drivers/gpu/host1x/job.c b/drivers/gpu/host1x/job.c
index c4e1050f2252..de5ec333ce1a 100644
--- a/drivers/gpu/host1x/job.c
+++ b/drivers/gpu/host1x/job.c
@@ -18,6 +18,7 @@
 
 #include <linux/dma-mapping.h>
 #include <linux/err.h>
+#include <linux/host1x.h>
 #include <linux/kref.h>
 #include <linux/module.h>
 #include <linux/scatterlist.h>
@@ -27,7 +28,6 @@
 
 #include "channel.h"
 #include "dev.h"
-#include "host1x_bo.h"
 #include "job.h"
 #include "syncpt.h"
 
@@ -264,7 +264,7 @@ static unsigned int do_relocs(struct host1x_job *job, struct host1x_bo *cmdbuf)
 }
 
 static bool check_reloc(struct host1x_reloc *reloc, struct host1x_bo *cmdbuf,
-                       unsigned int offset)
+                        unsigned int offset)
 {
         offset *= sizeof(u32);
 
@@ -281,7 +281,7 @@ struct host1x_firewall {
         unsigned int num_relocs;
         struct host1x_reloc *reloc;
 
-        struct host1x_bo *cmdbuf_id;
+        struct host1x_bo *cmdbuf;
         unsigned int offset;
 
         u32 words;
@@ -291,25 +291,37 @@ struct host1x_firewall {
         u32 count;
 };
 
+static int check_register(struct host1x_firewall *fw, unsigned long offset)
+{
+        if (fw->job->is_addr_reg(fw->dev, fw->class, offset)) {
+                if (!fw->num_relocs)
+                        return -EINVAL;
+
+                if (!check_reloc(fw->reloc, fw->cmdbuf, fw->offset))
+                        return -EINVAL;
+
+                fw->num_relocs--;
+                fw->reloc++;
+        }
+
+        return 0;
+}
+
 static int check_mask(struct host1x_firewall *fw)
 {
         u32 mask = fw->mask;
         u32 reg = fw->reg;
+        int ret;
 
         while (mask) {
                 if (fw->words == 0)
                         return -EINVAL;
 
                 if (mask & 1) {
-                        if (fw->job->is_addr_reg(fw->dev, fw->class, reg)) {
-                                if (!fw->num_relocs)
-                                        return -EINVAL;
-                                if (!check_reloc(fw->reloc, fw->cmdbuf_id,
-                                                 fw->offset))
-                                        return -EINVAL;
-                                fw->reloc++;
-                                fw->num_relocs--;
-                        }
+                        ret = check_register(fw, reg);
+                        if (ret < 0)
+                                return ret;
+
                         fw->words--;
                         fw->offset++;
                 }
@@ -324,19 +336,16 @@ static int check_incr(struct host1x_firewall *fw)
 {
         u32 count = fw->count;
         u32 reg = fw->reg;
+        int ret;
 
         while (count) {
                 if (fw->words == 0)
                         return -EINVAL;
 
-                if (fw->job->is_addr_reg(fw->dev, fw->class, reg)) {
-                        if (!fw->num_relocs)
-                                return -EINVAL;
-                        if (!check_reloc(fw->reloc, fw->cmdbuf_id, fw->offset))
-                                return -EINVAL;
-                        fw->reloc++;
-                        fw->num_relocs--;
-                }
+                ret = check_register(fw, reg);
+                if (ret < 0)
+                        return ret;
+
                 reg++;
                 fw->words--;
                 fw->offset++;
@@ -348,21 +357,17 @@ static int check_incr(struct host1x_firewall *fw)
 
 static int check_nonincr(struct host1x_firewall *fw)
 {
-        int is_addr_reg = fw->job->is_addr_reg(fw->dev, fw->class, fw->reg);
         u32 count = fw->count;
+        int ret;
 
         while (count) {
                 if (fw->words == 0)
                         return -EINVAL;
 
-                if (is_addr_reg) {
-                        if (!fw->num_relocs)
-                                return -EINVAL;
-                        if (!check_reloc(fw->reloc, fw->cmdbuf_id, fw->offset))
-                                return -EINVAL;
-                        fw->reloc++;
-                        fw->num_relocs--;
-                }
+                ret = check_register(fw, fw->reg);
+                if (ret < 0)
+                        return ret;
+
                 fw->words--;
                 fw->offset++;
                 count--;
@@ -381,7 +386,7 @@ static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g)
                 return 0;
 
         fw->words = g->words;
-        fw->cmdbuf_id = g->bo;
+        fw->cmdbuf = g->bo;
         fw->offset = 0;
 
         while (fw->words && !err) {
@@ -436,10 +441,6 @@ static int validate(struct host1x_firewall *fw, struct host1x_job_gather *g)
                 }
         }
 
-        /* No relocs should remain at this point */
-        if (fw->num_relocs)
-                err = -EINVAL;
-
 out:
         return err;
 }
@@ -493,6 +494,10 @@ static inline int copy_gathers(struct host1x_job *job, struct device *dev)
                 offset += g->words * sizeof(u32);
         }
 
+        /* No relocs should remain at this point */
+        if (fw.num_relocs)
+                return -EINVAL;
+
         return 0;
 }
 
diff --git a/drivers/gpu/host1x/job.h b/drivers/gpu/host1x/job.h
index fba45f20458e..33a697d6dcef 100644
--- a/drivers/gpu/host1x/job.h
+++ b/drivers/gpu/host1x/job.h
@@ -34,15 +34,6 @@ struct host1x_cmdbuf {
         u32 pad;
 };
 
-struct host1x_reloc {
-        struct host1x_bo *cmdbuf;
-        u32 cmdbuf_offset;
-        struct host1x_bo *target;
-        u32 target_offset;
-        u32 shift;
-        u32 pad;
-};
-
 struct host1x_waitchk {
         struct host1x_bo *bo;
         u32 offset;
@@ -56,105 +47,6 @@ struct host1x_job_unpin_data {
 };
 
 /*
- * Each submit is tracked as a host1x_job.
- */
-struct host1x_job {
-        /* When refcount goes to zero, job can be freed */
-        struct kref ref;
-
-        /* List entry */
-        struct list_head list;
-
-        /* Channel where job is submitted to */
-        struct host1x_channel *channel;
-
-        u32 client;
-
-        /* Gathers and their memory */
-        struct host1x_job_gather *gathers;
-        unsigned int num_gathers;
-
-        /* Wait checks to be processed at submit time */
-        struct host1x_waitchk *waitchk;
-        unsigned int num_waitchk;
-        u32 waitchk_mask;
-
-        /* Array of handles to be pinned & unpinned */
-        struct host1x_reloc *relocarray;
-        unsigned int num_relocs;
-        struct host1x_job_unpin_data *unpins;
-        unsigned int num_unpins;
-
-        dma_addr_t *addr_phys;
-        dma_addr_t *gather_addr_phys;
-        dma_addr_t *reloc_addr_phys;
-
-        /* Sync point id, number of increments and end related to the submit */
-        u32 syncpt_id;
-        u32 syncpt_incrs;
-        u32 syncpt_end;
-
-        /* Maximum time to wait for this job */
-        unsigned int timeout;
-
-        /* Index and number of slots used in the push buffer */
-        unsigned int first_get;
-        unsigned int num_slots;
-
-        /* Copy of gathers */
-        size_t gather_copy_size;
-        dma_addr_t gather_copy;
-        u8 *gather_copy_mapped;
-
-        /* Check if register is marked as an address reg */
-        int (*is_addr_reg)(struct device *dev, u32 reg, u32 class);
-
-        /* Request a SETCLASS to this class */
-        u32 class;
-
-        /* Add a channel wait for previous ops to complete */
-        bool serialize;
-};
-/*
- * Allocate memory for a job. Just enough memory will be allocated to
- * accomodate the submit.
- */
-struct host1x_job *host1x_job_alloc(struct host1x_channel *ch,
-                                    u32 num_cmdbufs, u32 num_relocs,
-                                    u32 num_waitchks);
-
-/*
- * Add a gather to a job.
- */
-void host1x_job_add_gather(struct host1x_job *job, struct host1x_bo *mem_id,
-                           u32 words, u32 offset);
-
-/*
- * Increment reference going to host1x_job.
- */
-struct host1x_job *host1x_job_get(struct host1x_job *job);
-
-/*
- * Decrement reference job, free if goes to zero.
- */
-void host1x_job_put(struct host1x_job *job);
-
-/*
- * Pin memory related to job. This handles relocation of addresses to the
- * host1x address space. Handles both the gather memory and any other memory
- * referred to from the gather buffers.
- *
- * Handles also patching out host waits that would wait for an expired sync
- * point value.
- */
-int host1x_job_pin(struct host1x_job *job, struct device *dev);
-
-/*
- * Unpin memory related to job.
- */
-void host1x_job_unpin(struct host1x_job *job);
-
-/*
  * Dump contents of job to debug output.
  */
 void host1x_job_dump(struct device *dev, struct host1x_job *job);
diff --git a/drivers/gpu/host1x/syncpt.c b/drivers/gpu/host1x/syncpt.c
index 409745b949db..159c479829c9 100644
--- a/drivers/gpu/host1x/syncpt.c
+++ b/drivers/gpu/host1x/syncpt.c
@@ -30,9 +30,32 @@
 #define SYNCPT_CHECK_PERIOD (2 * HZ)
 #define MAX_STUCK_CHECK_COUNT 15
 
-static struct host1x_syncpt *_host1x_syncpt_alloc(struct host1x *host,
-                                                  struct device *dev,
-                                                  bool client_managed)
+static struct host1x_syncpt_base *
+host1x_syncpt_base_request(struct host1x *host)
+{
+        struct host1x_syncpt_base *bases = host->bases;
+        unsigned int i;
+
+        for (i = 0; i < host->info->nb_bases; i++)
+                if (!bases[i].requested)
+                        break;
+
+        if (i >= host->info->nb_bases)
+                return NULL;
+
+        bases[i].requested = true;
+        return &bases[i];
+}
+
+static void host1x_syncpt_base_free(struct host1x_syncpt_base *base)
+{
+        if (base)
+                base->requested = false;
+}
+
+static struct host1x_syncpt *host1x_syncpt_alloc(struct host1x *host,
+                                                 struct device *dev,
+                                                 unsigned long flags)
 {
         int i;
         struct host1x_syncpt *sp = host->syncpt;
@@ -44,6 +67,12 @@ static struct host1x_syncpt *_host1x_syncpt_alloc(struct host1x *host,
         if (i >= host->info->nb_pts)
                 return NULL;
 
+        if (flags & HOST1X_SYNCPT_HAS_BASE) {
+                sp->base = host1x_syncpt_base_request(host);
+                if (!sp->base)
+                        return NULL;
+        }
+
         name = kasprintf(GFP_KERNEL, "%02d-%s", sp->id,
                         dev ? dev_name(dev) : NULL);
         if (!name)
@@ -51,7 +80,11 @@ static struct host1x_syncpt *_host1x_syncpt_alloc(struct host1x *host,
 
         sp->dev = dev;
         sp->name = name;
-        sp->client_managed = client_managed;
+
+        if (flags & HOST1X_SYNCPT_CLIENT_MANAGED)
+                sp->client_managed = true;
+        else
+                sp->client_managed = false;
 
         return sp;
 }
@@ -303,25 +336,35 @@ int host1x_syncpt_patch_wait(struct host1x_syncpt *sp, void *patch_addr)
 
 int host1x_syncpt_init(struct host1x *host)
 {
+        struct host1x_syncpt_base *bases;
         struct host1x_syncpt *syncpt;
         int i;
 
         syncpt = devm_kzalloc(host->dev, sizeof(*syncpt) * host->info->nb_pts,
-                GFP_KERNEL);
+                              GFP_KERNEL);
         if (!syncpt)
                 return -ENOMEM;
 
-        for (i = 0; i < host->info->nb_pts; ++i) {
+        bases = devm_kzalloc(host->dev, sizeof(*bases) * host->info->nb_bases,
+                             GFP_KERNEL);
+        if (!bases)
+                return -ENOMEM;
+
+        for (i = 0; i < host->info->nb_pts; i++) {
                 syncpt[i].id = i;
                 syncpt[i].host = host;
         }
 
+        for (i = 0; i < host->info->nb_bases; i++)
+                bases[i].id = i;
+
         host->syncpt = syncpt;
+        host->bases = bases;
 
         host1x_syncpt_restore(host);
 
         /* Allocate sync point to use for clearing waits for expired fences */
-        host->nop_sp = _host1x_syncpt_alloc(host, NULL, false);
+        host->nop_sp = host1x_syncpt_alloc(host, NULL, 0);
         if (!host->nop_sp)
                 return -ENOMEM;
 
@@ -329,10 +372,10 @@ int host1x_syncpt_init(struct host1x *host)
 }
 
 struct host1x_syncpt *host1x_syncpt_request(struct device *dev,
-                                            bool client_managed)
+                                            unsigned long flags)
 {
         struct host1x *host = dev_get_drvdata(dev->parent);
-        return _host1x_syncpt_alloc(host, dev, client_managed);
+        return host1x_syncpt_alloc(host, dev, flags);
 }
 
 void host1x_syncpt_free(struct host1x_syncpt *sp)
@@ -340,7 +383,9 @@ void host1x_syncpt_free(struct host1x_syncpt *sp)
         if (!sp)
                 return;
 
+        host1x_syncpt_base_free(sp->base);
         kfree(sp->name);
+        sp->base = NULL;
         sp->dev = NULL;
         sp->name = NULL;
         sp->client_managed = false;
@@ -354,6 +399,25 @@ void host1x_syncpt_deinit(struct host1x *host)
                 kfree(sp->name);
 }
 
+/*
+ * Read max. It indicates how many operations there are in queue, either in
+ * channel or in a software thread.
+ * */
+u32 host1x_syncpt_read_max(struct host1x_syncpt *sp)
+{
+        smp_rmb();
+        return (u32)atomic_read(&sp->max_val);
+}
+
+/*
+ * Read min, which is a shadow of the current sync point value in hardware.
+ */
+u32 host1x_syncpt_read_min(struct host1x_syncpt *sp)
+{
+        smp_rmb();
+        return (u32)atomic_read(&sp->min_val);
+}
+
 int host1x_syncpt_nb_pts(struct host1x *host)
 {
         return host->info->nb_pts;
@@ -375,3 +439,13 @@ struct host1x_syncpt *host1x_syncpt_get(struct host1x *host, u32 id)
                 return NULL;
         return host->syncpt + id;
 }
+
+struct host1x_syncpt_base *host1x_syncpt_get_base(struct host1x_syncpt *sp)
+{
+        return sp ? sp->base : NULL;
+}
+
+u32 host1x_syncpt_base_id(struct host1x_syncpt_base *base)
+{
+        return base->id;
+}
diff --git a/drivers/gpu/host1x/syncpt.h b/drivers/gpu/host1x/syncpt.h
index 267c0b9d3647..9056465ecd3f 100644
--- a/drivers/gpu/host1x/syncpt.h
+++ b/drivers/gpu/host1x/syncpt.h
@@ -20,6 +20,7 @@
 #define __HOST1X_SYNCPT_H
 
 #include <linux/atomic.h>
+#include <linux/host1x.h>
 #include <linux/kernel.h>
 #include <linux/sched.h>
 
@@ -30,6 +31,11 @@ struct host1x;
 /* Reserved for replacing an expired wait with a NOP */
 #define HOST1X_SYNCPT_RESERVED                  0
 
+struct host1x_syncpt_base {
+        unsigned int id;
+        bool requested;
+};
+
 struct host1x_syncpt {
         int id;
         atomic_t min_val;
@@ -39,6 +45,7 @@ struct host1x_syncpt {
         bool client_managed;
         struct host1x *host;
         struct device *dev;
+        struct host1x_syncpt_base *base;
 
         /* interrupt data */
         struct host1x_syncpt_intr intr;
@@ -50,25 +57,6 @@ int host1x_syncpt_init(struct host1x *host);
 /*  Free sync point array */
 void host1x_syncpt_deinit(struct host1x *host);
 
-/*
- * Read max. It indicates how many operations there are in queue, either in
- * channel or in a software thread.
- * */
-static inline u32 host1x_syncpt_read_max(struct host1x_syncpt *sp)
-{
-        smp_rmb();
-        return (u32)atomic_read(&sp->max_val);
-}
-
-/*
- * Read min, which is a shadow of the current sync point value in hardware.
- */
-static inline u32 host1x_syncpt_read_min(struct host1x_syncpt *sp)
-{
-        smp_rmb();
-        return (u32)atomic_read(&sp->min_val);
-}
-
 /* Return number of sync point supported. */
 int host1x_syncpt_nb_pts(struct host1x *host);
 
@@ -112,9 +100,6 @@ static inline bool host1x_syncpt_idle(struct host1x_syncpt *sp)
         return (min == max);
 }
 
-/* Return pointer to struct denoting sync point id. */
-struct host1x_syncpt *host1x_syncpt_get(struct host1x *host, u32 id);
-
 /* Load current value from hardware to the shadow register. */
 u32 host1x_syncpt_load(struct host1x_syncpt *sp);
 
@@ -130,16 +115,9 @@ void host1x_syncpt_restore(struct host1x *host);
 /* Read current wait base value into shadow register and return it. */
 u32 host1x_syncpt_load_wait_base(struct host1x_syncpt *sp);
 
-/* Request incrementing a sync point. */
-int host1x_syncpt_incr(struct host1x_syncpt *sp);
-
 /* Indicate future operations by incrementing the sync point max. */
 u32 host1x_syncpt_incr_max(struct host1x_syncpt *sp, u32 incrs);
 
-/* Wait until sync point reaches a threshold value, or a timeout. */
-int host1x_syncpt_wait(struct host1x_syncpt *sp, u32 thresh,
-                        long timeout, u32 *value);
-
 /* Check if sync point id is valid. */
 static inline int host1x_syncpt_is_valid(struct host1x_syncpt *sp)
 {
@@ -149,14 +127,4 @@ static inline int host1x_syncpt_is_valid(struct host1x_syncpt *sp)
 /* Patch a wait by replacing it with a wait for syncpt 0 value 0 */
 int host1x_syncpt_patch_wait(struct host1x_syncpt *sp, void *patch_addr);
 
-/* Return id of the sync point */
-u32 host1x_syncpt_id(struct host1x_syncpt *sp);
-
-/* Allocate a sync point for a device. */
-struct host1x_syncpt *host1x_syncpt_request(struct device *dev,
-                                            bool client_managed);
-
-/* Free a sync point. */
-void host1x_syncpt_free(struct host1x_syncpt *sp);
-
 #endif