Merge branch 'drm-next' of git://people.freedesktop.org/~airlied/linux

Pull drm updates from Dave Airlie:
 "Highlights:

   - drm:

     Generic display port aux features, primary plane support, drm
     master management fixes, logging cleanups, enforced locking checks
     (instead of docs), documentation improvements, minor number
     handling cleanup, pseudofs for shared inodes.

   - ttm:

     add ability to allocate from both ends

   - i915:

     broadwell features, power domain and runtime pm, per-process
     address space infrastructure (not enabled)

   - msm:

     power management, hdmi audio support

   - nouveau:

     ongoing GPU fault recovery, initial maxwell support, random fixes

   - exynos:

     refactored driver to clean up a lot of abstraction, DP support
     moved into drm, LVDS bridge support added, parallel panel support

   - gma500:

     SGX MMU support, SGX irq handling, asle irq work fixes

   - radeon:

     video engine bringup, ring handling fixes, use dp aux helpers

   - vmwgfx:

     add rendernode support"

* 'drm-next' of git://people.freedesktop.org/~airlied/linux: (849 commits)
  DRM: armada: fix corruption while loading cursors
  drm/dp_helper: don't return EPROTO for defers (v2)
  drm/bridge: export ptn3460_init function
  drm/exynos: remove MODULE_DEVICE_TABLE definitions
  ARM: dts: exynos4412-trats2: enable exynos/fimd node
  ARM: dts: exynos4210-trats: enable exynos/fimd node
  ARM: dts: exynos4412-trats2: add panel node
  ARM: dts: exynos4210-trats: add panel node
  ARM: dts: exynos4: add MIPI DSI Master node
  drm/panel: add S6E8AA0 driver
  ARM: dts: exynos4210-universal_c210: add proper panel node
  drm/panel: add ld9040 driver
  panel/ld9040: add DT bindings
  panel/s6e8aa0: add DT bindings
  drm/exynos: add DSIM driver
  exynos/dsim: add DT bindings
  drm/exynos: disallow fbdev initialization if no device is connected
  drm/mipi_dsi: create dsi devices only for nodes with reg property
  drm/mipi_dsi: add flags to DSI messages
  Skip intel_crt_init for Dell XPS 8700
  ...

441 files changed, 35607 insertions, 15287 deletions

diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig
index 8e7fa4dbaed8..d1cc2f613a78 100644
--- a/drivers/gpu/drm/Kconfig
+++ b/drivers/gpu/drm/Kconfig
@@ -199,3 +199,5 @@ source "drivers/gpu/drm/msm/Kconfig"
 source "drivers/gpu/drm/tegra/Kconfig"
 
 source "drivers/gpu/drm/panel/Kconfig"
+
+source "drivers/gpu/drm/bridge/Kconfig"
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index 292a79d64146..9d25dbbe6771 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -13,7 +13,8 @@ drm-y       :=	drm_auth.o drm_buffer.o drm_bufs.o drm_cache.o \
                 drm_crtc.o drm_modes.o drm_edid.o \
                 drm_info.o drm_debugfs.o drm_encoder_slave.o \
                 drm_trace_points.o drm_global.o drm_prime.o \
-                drm_rect.o drm_vma_manager.o drm_flip_work.o
+                drm_rect.o drm_vma_manager.o drm_flip_work.o \
+                drm_plane_helper.o
 
 drm-$(CONFIG_COMPAT) += drm_ioc32.o
 drm-$(CONFIG_DRM_GEM_CMA_HELPER) += drm_gem_cma_helper.o
@@ -63,3 +64,4 @@ obj-$(CONFIG_DRM_MSM) += msm/
 obj-$(CONFIG_DRM_TEGRA) += tegra/
 obj-y                   += i2c/
 obj-y                   += panel/
+obj-y                   += bridge/
diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c
index d8e398275ca8..81c34f949dfc 100644
--- a/drivers/gpu/drm/armada/armada_crtc.c
+++ b/drivers/gpu/drm/armada/armada_crtc.c
@@ -478,11 +478,12 @@ static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
         unsigned i;
         bool interlaced;
 
-        drm_framebuffer_reference(crtc->fb);
+        drm_framebuffer_reference(crtc->primary->fb);
 
         interlaced = !!(adj->flags & DRM_MODE_FLAG_INTERLACE);
 
-        i = armada_drm_crtc_calc_fb(dcrtc->crtc.fb, x, y, regs, interlaced);
+        i = armada_drm_crtc_calc_fb(dcrtc->crtc.primary->fb,
+                                    x, y, regs, interlaced);
 
         rm = adj->crtc_hsync_start - adj->crtc_hdisplay;
         lm = adj->crtc_htotal - adj->crtc_hsync_end;
@@ -567,10 +568,10 @@ static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
         }
 
         val = CFG_GRA_ENA | CFG_GRA_HSMOOTH;
-        val |= CFG_GRA_FMT(drm_fb_to_armada_fb(dcrtc->crtc.fb)->fmt);
-        val |= CFG_GRA_MOD(drm_fb_to_armada_fb(dcrtc->crtc.fb)->mod);
+        val |= CFG_GRA_FMT(drm_fb_to_armada_fb(dcrtc->crtc.primary->fb)->fmt);
+        val |= CFG_GRA_MOD(drm_fb_to_armada_fb(dcrtc->crtc.primary->fb)->mod);
 
-        if (drm_fb_to_armada_fb(dcrtc->crtc.fb)->fmt > CFG_420)
+        if (drm_fb_to_armada_fb(dcrtc->crtc.primary->fb)->fmt > CFG_420)
                 val |= CFG_PALETTE_ENA;
 
         if (interlaced)
@@ -608,7 +609,7 @@ static int armada_drm_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
         struct armada_regs regs[4];
         unsigned i;
 
-        i = armada_drm_crtc_calc_fb(crtc->fb, crtc->x, crtc->y, regs,
+        i = armada_drm_crtc_calc_fb(crtc->primary->fb, crtc->x, crtc->y, regs,
                                     dcrtc->interlaced);
         armada_reg_queue_end(regs, i);
 
@@ -616,7 +617,7 @@ static int armada_drm_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
         wait_event(dcrtc->frame_wait, !dcrtc->frame_work);
 
         /* Take a reference to the new fb as we're using it */
-        drm_framebuffer_reference(crtc->fb);
+        drm_framebuffer_reference(crtc->primary->fb);
 
         /* Update the base in the CRTC */
         armada_drm_crtc_update_regs(dcrtc, regs);
@@ -637,7 +638,7 @@ static void armada_drm_crtc_disable(struct drm_crtc *crtc)
         struct armada_crtc *dcrtc = drm_to_armada_crtc(crtc);
 
         armada_drm_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
-        armada_drm_crtc_finish_fb(dcrtc, crtc->fb, true);
+        armada_drm_crtc_finish_fb(dcrtc, crtc->primary->fb, true);
 
         /* Power down most RAMs and FIFOs */
         writel_relaxed(CFG_PDWN256x32 | CFG_PDWN256x24 | CFG_PDWN256x8 |
@@ -678,6 +679,7 @@ static void armada_load_cursor_argb(void __iomem *base, uint32_t *pix,
                                        base + LCD_SPU_SRAM_WRDAT);
                         writel_relaxed(addr | SRAM_WRITE,
                                        base + LCD_SPU_SRAM_CTRL);
+                        readl_relaxed(base + LCD_SPU_HWC_OVSA_HPXL_VLN);
                         addr += 1;
                         if ((addr & 0x00ff) == 0)
                                 addr += 0xf00;
@@ -904,7 +906,7 @@ static int armada_drm_crtc_page_flip(struct drm_crtc *crtc,
         int ret;
 
         /* We don't support changing the pixel format */
-        if (fb->pixel_format != crtc->fb->pixel_format)
+        if (fb->pixel_format != crtc->primary->fb->pixel_format)
                 return -EINVAL;
 
         work = kmalloc(sizeof(*work), GFP_KERNEL);
@@ -912,7 +914,7 @@ static int armada_drm_crtc_page_flip(struct drm_crtc *crtc,
                 return -ENOMEM;
 
         work->event = event;
-        work->old_fb = dcrtc->crtc.fb;
+        work->old_fb = dcrtc->crtc.primary->fb;
 
         i = armada_drm_crtc_calc_fb(fb, crtc->x, crtc->y, work->regs,
                                     dcrtc->interlaced);
@@ -941,7 +943,7 @@ static int armada_drm_crtc_page_flip(struct drm_crtc *crtc,
          * will _not_ drop that reference on successful return from this
          * function.  Simply mark this new framebuffer as the current one.
          */
-        dcrtc->crtc.fb = fb;
+        dcrtc->crtc.primary->fb = fb;
 
         /*
          * Finally, if the display is blanked, we won't receive an
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index cca063b11083..a4afdc8bb578 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -81,7 +81,7 @@ static bool ast_get_vbios_mode_info(struct drm_crtc *crtc, struct drm_display_mo
         u32 refresh_rate_index = 0, mode_id, color_index, refresh_rate;
         u32 hborder, vborder;
 
-        switch (crtc->fb->bits_per_pixel) {
+        switch (crtc->primary->fb->bits_per_pixel) {
         case 8:
                 vbios_mode->std_table = &vbios_stdtable[VGAModeIndex];
                 color_index = VGAModeIndex - 1;
@@ -176,7 +176,7 @@ static bool ast_get_vbios_mode_info(struct drm_crtc *crtc, struct drm_display_mo
                 ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0x8e, mode_id & 0xff);
 
                 ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0x91, 0xa8);
-                ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0x92, crtc->fb->bits_per_pixel);
+                ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0x92, crtc->primary->fb->bits_per_pixel);
                 ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0x93, adjusted_mode->clock / 1000);
                 ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0x94, adjusted_mode->crtc_hdisplay);
                 ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0x95, adjusted_mode->crtc_hdisplay >> 8);
@@ -340,7 +340,7 @@ static void ast_set_offset_reg(struct drm_crtc *crtc)
 
         u16 offset;
 
-        offset = crtc->fb->pitches[0] >> 3;
+        offset = crtc->primary->fb->pitches[0] >> 3;
         ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0x13, (offset & 0xff));
         ast_set_index_reg(ast, AST_IO_CRTC_PORT, 0xb0, (offset >> 8) & 0x3f);
 }
@@ -365,7 +365,7 @@ static void ast_set_ext_reg(struct drm_crtc *crtc, struct drm_display_mode *mode
         struct ast_private *ast = crtc->dev->dev_private;
         u8 jregA0 = 0, jregA3 = 0, jregA8 = 0;
 
-        switch (crtc->fb->bits_per_pixel) {
+        switch (crtc->primary->fb->bits_per_pixel) {
         case 8:
                 jregA0 = 0x70;
                 jregA3 = 0x01;
@@ -418,7 +418,7 @@ static void ast_set_sync_reg(struct drm_device *dev, struct drm_display_mode *mo
 static bool ast_set_dac_reg(struct drm_crtc *crtc, struct drm_display_mode *mode,
                      struct ast_vbios_mode_info *vbios_mode)
 {
-        switch (crtc->fb->bits_per_pixel) {
+        switch (crtc->primary->fb->bits_per_pixel) {
         case 8:
                 break;
         default:
@@ -490,7 +490,7 @@ static int ast_crtc_do_set_base(struct drm_crtc *crtc,
                 ast_bo_unreserve(bo);
         }
 
-        ast_fb = to_ast_framebuffer(crtc->fb);
+        ast_fb = to_ast_framebuffer(crtc->primary->fb);
         obj = ast_fb->obj;
         bo = gem_to_ast_bo(obj);
 
diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c
index 4ea9b17ac17a..b8246227bab0 100644
--- a/drivers/gpu/drm/ast/ast_ttm.c
+++ b/drivers/gpu/drm/ast/ast_ttm.c
@@ -259,7 +259,9 @@ int ast_mm_init(struct ast_private *ast)
 
         ret = ttm_bo_device_init(&ast->ttm.bdev,
                                  ast->ttm.bo_global_ref.ref.object,
-                                 &ast_bo_driver, DRM_FILE_PAGE_OFFSET,
+                                 &ast_bo_driver,
+                                 dev->anon_inode->i_mapping,
+                                 DRM_FILE_PAGE_OFFSET,
                                  true);
         if (ret) {
                 DRM_ERROR("Error initialising bo driver; %d\n", ret);
@@ -324,7 +326,6 @@ int ast_bo_create(struct drm_device *dev, int size, int align,
         }
 
         astbo->bo.bdev = &ast->ttm.bdev;
-        astbo->bo.bdev->dev_mapping = dev->dev_mapping;
 
         ast_ttm_placement(astbo, TTM_PL_FLAG_VRAM | TTM_PL_FLAG_SYSTEM);
 
diff --git a/drivers/gpu/drm/bochs/bochs_kms.c b/drivers/gpu/drm/bochs/bochs_kms.c
index 62ec7d4b3816..dcf2e55f4ae9 100644
--- a/drivers/gpu/drm/bochs/bochs_kms.c
+++ b/drivers/gpu/drm/bochs/bochs_kms.c
@@ -62,10 +62,10 @@ static int bochs_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
                 }
         }
 
-        if (WARN_ON(crtc->fb == NULL))
+        if (WARN_ON(crtc->primary->fb == NULL))
                 return -EINVAL;
 
-        bochs_fb = to_bochs_framebuffer(crtc->fb);
+        bochs_fb = to_bochs_framebuffer(crtc->primary->fb);
         bo = gem_to_bochs_bo(bochs_fb->obj);
         ret = ttm_bo_reserve(&bo->bo, true, false, false, 0);
         if (ret)
diff --git a/drivers/gpu/drm/bochs/bochs_mm.c b/drivers/gpu/drm/bochs/bochs_mm.c
index ce6858765b37..f488be55d650 100644
--- a/drivers/gpu/drm/bochs/bochs_mm.c
+++ b/drivers/gpu/drm/bochs/bochs_mm.c
@@ -225,7 +225,9 @@ int bochs_mm_init(struct bochs_device *bochs)
 
         ret = ttm_bo_device_init(&bochs->ttm.bdev,
                                  bochs->ttm.bo_global_ref.ref.object,
-                                 &bochs_bo_driver, DRM_FILE_PAGE_OFFSET,
+                                 &bochs_bo_driver,
+                                 bochs->dev->anon_inode->i_mapping,
+                                 DRM_FILE_PAGE_OFFSET,
                                  true);
         if (ret) {
                 DRM_ERROR("Error initialising bo driver; %d\n", ret);
@@ -359,7 +361,7 @@ static int bochs_bo_create(struct drm_device *dev, int size, int align,
         }
 
         bochsbo->bo.bdev = &bochs->ttm.bdev;
-        bochsbo->bo.bdev->dev_mapping = dev->dev_mapping;
+        bochsbo->bo.bdev->dev_mapping = dev->anon_inode->i_mapping;
 
         bochs_ttm_placement(bochsbo, TTM_PL_FLAG_VRAM | TTM_PL_FLAG_SYSTEM);
 
diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig
new file mode 100644
index 000000000000..884923f982d9
--- /dev/null
+++ b/drivers/gpu/drm/bridge/Kconfig
@@ -0,0 +1,5 @@
+config DRM_PTN3460
+        tristate "PTN3460 DP/LVDS bridge"
+        depends on DRM
+        select DRM_KMS_HELPER
+        ---help---
diff --git a/drivers/gpu/drm/bridge/Makefile b/drivers/gpu/drm/bridge/Makefile
new file mode 100644
index 000000000000..b4733e1fbd2e
--- /dev/null
+++ b/drivers/gpu/drm/bridge/Makefile
@@ -0,0 +1,3 @@
+ccflags-y := -Iinclude/drm
+
+obj-$(CONFIG_DRM_PTN3460) += ptn3460.o
diff --git a/drivers/gpu/drm/bridge/ptn3460.c b/drivers/gpu/drm/bridge/ptn3460.c
new file mode 100644
index 000000000000..b171901a3553
--- /dev/null
+++ b/drivers/gpu/drm/bridge/ptn3460.c
@@ -0,0 +1,350 @@
+/*
+ * NXP PTN3460 DP/LVDS bridge driver
+ *
+ * Copyright (C) 2013 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+#include <linux/i2c.h>
+#include <linux/gpio.h>
+#include <linux/delay.h>
+
+#include "drmP.h"
+#include "drm_edid.h"
+#include "drm_crtc.h"
+#include "drm_crtc_helper.h"
+
+#include "bridge/ptn3460.h"
+
+#define PTN3460_EDID_ADDR                       0x0
+#define PTN3460_EDID_EMULATION_ADDR             0x84
+#define PTN3460_EDID_ENABLE_EMULATION           0
+#define PTN3460_EDID_EMULATION_SELECTION        1
+#define PTN3460_EDID_SRAM_LOAD_ADDR             0x85
+
+struct ptn3460_bridge {
+        struct drm_connector connector;
+        struct i2c_client *client;
+        struct drm_encoder *encoder;
+        struct drm_bridge *bridge;
+        struct edid *edid;
+        int gpio_pd_n;
+        int gpio_rst_n;
+        u32 edid_emulation;
+        bool enabled;
+};
+
+static int ptn3460_read_bytes(struct ptn3460_bridge *ptn_bridge, char addr,
+                u8 *buf, int len)
+{
+        int ret;
+
+        ret = i2c_master_send(ptn_bridge->client, &addr, 1);
+        if (ret <= 0) {
+                DRM_ERROR("Failed to send i2c command, ret=%d\n", ret);
+                return ret;
+        }
+
+        ret = i2c_master_recv(ptn_bridge->client, buf, len);
+        if (ret <= 0) {
+                DRM_ERROR("Failed to recv i2c data, ret=%d\n", ret);
+                return ret;
+        }
+
+        return 0;
+}
+
+static int ptn3460_write_byte(struct ptn3460_bridge *ptn_bridge, char addr,
+                char val)
+{
+        int ret;
+        char buf[2];
+
+        buf[0] = addr;
+        buf[1] = val;
+
+        ret = i2c_master_send(ptn_bridge->client, buf, ARRAY_SIZE(buf));
+        if (ret <= 0) {
+                DRM_ERROR("Failed to send i2c command, ret=%d\n", ret);
+                return ret;
+        }
+
+        return 0;
+}
+
+static int ptn3460_select_edid(struct ptn3460_bridge *ptn_bridge)
+{
+        int ret;
+        char val;
+
+        /* Load the selected edid into SRAM (accessed at PTN3460_EDID_ADDR) */
+        ret = ptn3460_write_byte(ptn_bridge, PTN3460_EDID_SRAM_LOAD_ADDR,
+                        ptn_bridge->edid_emulation);
+        if (ret) {
+                DRM_ERROR("Failed to transfer edid to sram, ret=%d\n", ret);
+                return ret;
+        }
+
+        /* Enable EDID emulation and select the desired EDID */
+        val = 1 << PTN3460_EDID_ENABLE_EMULATION |
+                ptn_bridge->edid_emulation << PTN3460_EDID_EMULATION_SELECTION;
+
+        ret = ptn3460_write_byte(ptn_bridge, PTN3460_EDID_EMULATION_ADDR, val);
+        if (ret) {
+                DRM_ERROR("Failed to write edid value, ret=%d\n", ret);
+                return ret;
+        }
+
+        return 0;
+}
+
+static void ptn3460_pre_enable(struct drm_bridge *bridge)
+{
+        struct ptn3460_bridge *ptn_bridge = bridge->driver_private;
+        int ret;
+
+        if (ptn_bridge->enabled)
+                return;
+
+        if (gpio_is_valid(ptn_bridge->gpio_pd_n))
+                gpio_set_value(ptn_bridge->gpio_pd_n, 1);
+
+        if (gpio_is_valid(ptn_bridge->gpio_rst_n)) {
+                gpio_set_value(ptn_bridge->gpio_rst_n, 0);
+                udelay(10);
+                gpio_set_value(ptn_bridge->gpio_rst_n, 1);
+        }
+
+        /*
+         * There's a bug in the PTN chip where it falsely asserts hotplug before
+         * it is fully functional. We're forced to wait for the maximum start up
+         * time specified in the chip's datasheet to make sure we're really up.
+         */
+        msleep(90);
+
+        ret = ptn3460_select_edid(ptn_bridge);
+        if (ret)
+                DRM_ERROR("Select edid failed ret=%d\n", ret);
+
+        ptn_bridge->enabled = true;
+}
+
+static void ptn3460_enable(struct drm_bridge *bridge)
+{
+}
+
+static void ptn3460_disable(struct drm_bridge *bridge)
+{
+        struct ptn3460_bridge *ptn_bridge = bridge->driver_private;
+
+        if (!ptn_bridge->enabled)
+                return;
+
+        ptn_bridge->enabled = false;
+
+        if (gpio_is_valid(ptn_bridge->gpio_rst_n))
+                gpio_set_value(ptn_bridge->gpio_rst_n, 1);
+
+        if (gpio_is_valid(ptn_bridge->gpio_pd_n))
+                gpio_set_value(ptn_bridge->gpio_pd_n, 0);
+}
+
+static void ptn3460_post_disable(struct drm_bridge *bridge)
+{
+}
+
+void ptn3460_bridge_destroy(struct drm_bridge *bridge)
+{
+        struct ptn3460_bridge *ptn_bridge = bridge->driver_private;
+
+        drm_bridge_cleanup(bridge);
+        if (gpio_is_valid(ptn_bridge->gpio_pd_n))
+                gpio_free(ptn_bridge->gpio_pd_n);
+        if (gpio_is_valid(ptn_bridge->gpio_rst_n))
+                gpio_free(ptn_bridge->gpio_rst_n);
+        /* Nothing else to free, we've got devm allocated memory */
+}
+
+struct drm_bridge_funcs ptn3460_bridge_funcs = {
+        .pre_enable = ptn3460_pre_enable,
+        .enable = ptn3460_enable,
+        .disable = ptn3460_disable,
+        .post_disable = ptn3460_post_disable,
+        .destroy = ptn3460_bridge_destroy,
+};
+
+int ptn3460_get_modes(struct drm_connector *connector)
+{
+        struct ptn3460_bridge *ptn_bridge;
+        u8 *edid;
+        int ret, num_modes;
+        bool power_off;
+
+        ptn_bridge = container_of(connector, struct ptn3460_bridge, connector);
+
+        if (ptn_bridge->edid)
+                return drm_add_edid_modes(connector, ptn_bridge->edid);
+
+        power_off = !ptn_bridge->enabled;
+        ptn3460_pre_enable(ptn_bridge->bridge);
+
+        edid = kmalloc(EDID_LENGTH, GFP_KERNEL);
+        if (!edid) {
+                DRM_ERROR("Failed to allocate edid\n");
+                return 0;
+        }
+
+        ret = ptn3460_read_bytes(ptn_bridge, PTN3460_EDID_ADDR, edid,
+                        EDID_LENGTH);
+        if (ret) {
+                kfree(edid);
+                num_modes = 0;
+                goto out;
+        }
+
+        ptn_bridge->edid = (struct edid *)edid;
+        drm_mode_connector_update_edid_property(connector, ptn_bridge->edid);
+
+        num_modes = drm_add_edid_modes(connector, ptn_bridge->edid);
+
+out:
+        if (power_off)
+                ptn3460_disable(ptn_bridge->bridge);
+
+        return num_modes;
+}
+
+static int ptn3460_mode_valid(struct drm_connector *connector,
+                struct drm_display_mode *mode)
+{
+        return MODE_OK;
+}
+
+struct drm_encoder *ptn3460_best_encoder(struct drm_connector *connector)
+{
+        struct ptn3460_bridge *ptn_bridge;
+
+        ptn_bridge = container_of(connector, struct ptn3460_bridge, connector);
+
+        return ptn_bridge->encoder;
+}
+
+struct drm_connector_helper_funcs ptn3460_connector_helper_funcs = {
+        .get_modes = ptn3460_get_modes,
+        .mode_valid = ptn3460_mode_valid,
+        .best_encoder = ptn3460_best_encoder,
+};
+
+enum drm_connector_status ptn3460_detect(struct drm_connector *connector,
+                bool force)
+{
+        return connector_status_connected;
+}
+
+void ptn3460_connector_destroy(struct drm_connector *connector)
+{
+        drm_connector_cleanup(connector);
+}
+
+struct drm_connector_funcs ptn3460_connector_funcs = {
+        .dpms = drm_helper_connector_dpms,
+        .fill_modes = drm_helper_probe_single_connector_modes,
+        .detect = ptn3460_detect,
+        .destroy = ptn3460_connector_destroy,
+};
+
+int ptn3460_init(struct drm_device *dev, struct drm_encoder *encoder,
+                struct i2c_client *client, struct device_node *node)
+{
+        int ret;
+        struct drm_bridge *bridge;
+        struct ptn3460_bridge *ptn_bridge;
+
+        bridge = devm_kzalloc(dev->dev, sizeof(*bridge), GFP_KERNEL);
+        if (!bridge) {
+                DRM_ERROR("Failed to allocate drm bridge\n");
+                return -ENOMEM;
+        }
+
+        ptn_bridge = devm_kzalloc(dev->dev, sizeof(*ptn_bridge), GFP_KERNEL);
+        if (!ptn_bridge) {
+                DRM_ERROR("Failed to allocate ptn bridge\n");
+                return -ENOMEM;
+        }
+
+        ptn_bridge->client = client;
+        ptn_bridge->encoder = encoder;
+        ptn_bridge->bridge = bridge;
+        ptn_bridge->gpio_pd_n = of_get_named_gpio(node, "powerdown-gpio", 0);
+        if (gpio_is_valid(ptn_bridge->gpio_pd_n)) {
+                ret = gpio_request_one(ptn_bridge->gpio_pd_n,
+                                GPIOF_OUT_INIT_HIGH, "PTN3460_PD_N");
+                if (ret) {
+                        DRM_ERROR("Request powerdown-gpio failed (%d)\n", ret);
+                        return ret;
+                }
+        }
+
+        ptn_bridge->gpio_rst_n = of_get_named_gpio(node, "reset-gpio", 0);
+        if (gpio_is_valid(ptn_bridge->gpio_rst_n)) {
+                /*
+                 * Request the reset pin low to avoid the bridge being
+                 * initialized prematurely
+                 */
+                ret = gpio_request_one(ptn_bridge->gpio_rst_n,
+                                GPIOF_OUT_INIT_LOW, "PTN3460_RST_N");
+                if (ret) {
+                        DRM_ERROR("Request reset-gpio failed (%d)\n", ret);
+                        gpio_free(ptn_bridge->gpio_pd_n);
+                        return ret;
+                }
+        }
+
+        ret = of_property_read_u32(node, "edid-emulation",
+                        &ptn_bridge->edid_emulation);
+        if (ret) {
+                DRM_ERROR("Can't read edid emulation value\n");
+                goto err;
+        }
+
+        ret = drm_bridge_init(dev, bridge, &ptn3460_bridge_funcs);
+        if (ret) {
+                DRM_ERROR("Failed to initialize bridge with drm\n");
+                goto err;
+        }
+
+        bridge->driver_private = ptn_bridge;
+        encoder->bridge = bridge;
+
+        ret = drm_connector_init(dev, &ptn_bridge->connector,
+                        &ptn3460_connector_funcs, DRM_MODE_CONNECTOR_LVDS);
+        if (ret) {
+                DRM_ERROR("Failed to initialize connector with drm\n");
+                goto err;
+        }
+        drm_connector_helper_add(&ptn_bridge->connector,
+                        &ptn3460_connector_helper_funcs);
+        drm_sysfs_connector_add(&ptn_bridge->connector);
+        drm_mode_connector_attach_encoder(&ptn_bridge->connector, encoder);
+
+        return 0;
+
+err:
+        if (gpio_is_valid(ptn_bridge->gpio_pd_n))
+                gpio_free(ptn_bridge->gpio_pd_n);
+        if (gpio_is_valid(ptn_bridge->gpio_rst_n))
+                gpio_free(ptn_bridge->gpio_rst_n);
+        return ret;
+}
+EXPORT_SYMBOL(ptn3460_init);
diff --git a/drivers/gpu/drm/cirrus/cirrus_mode.c b/drivers/gpu/drm/cirrus/cirrus_mode.c
index 530f78f84dee..2d64aea83df2 100644
--- a/drivers/gpu/drm/cirrus/cirrus_mode.c
+++ b/drivers/gpu/drm/cirrus/cirrus_mode.c
@@ -149,7 +149,7 @@ static int cirrus_crtc_do_set_base(struct drm_crtc *crtc,
                 cirrus_bo_unreserve(bo);
         }
 
-        cirrus_fb = to_cirrus_framebuffer(crtc->fb);
+        cirrus_fb = to_cirrus_framebuffer(crtc->primary->fb);
         obj = cirrus_fb->obj;
         bo = gem_to_cirrus_bo(obj);
 
@@ -268,7 +268,7 @@ static int cirrus_crtc_mode_set(struct drm_crtc *crtc,
         sr07 = RREG8(SEQ_DATA);
         sr07 &= 0xe0;
         hdr = 0;
-        switch (crtc->fb->bits_per_pixel) {
+        switch (crtc->primary->fb->bits_per_pixel) {
         case 8:
                 sr07 |= 0x11;
                 break;
@@ -291,13 +291,13 @@ static int cirrus_crtc_mode_set(struct drm_crtc *crtc,
         WREG_SEQ(0x7, sr07);
 
         /* Program the pitch */
-        tmp = crtc->fb->pitches[0] / 8;
+        tmp = crtc->primary->fb->pitches[0] / 8;
         WREG_CRT(VGA_CRTC_OFFSET, tmp);
 
         /* Enable extended blanking and pitch bits, and enable full memory */
         tmp = 0x22;
-        tmp |= (crtc->fb->pitches[0] >> 7) & 0x10;
-        tmp |= (crtc->fb->pitches[0] >> 6) & 0x40;
+        tmp |= (crtc->primary->fb->pitches[0] >> 7) & 0x10;
+        tmp |= (crtc->primary->fb->pitches[0] >> 6) & 0x40;
         WREG_CRT(0x1b, tmp);
 
         /* Enable high-colour modes */
diff --git a/drivers/gpu/drm/cirrus/cirrus_ttm.c b/drivers/gpu/drm/cirrus/cirrus_ttm.c
index 8b37c25ff9bd..92e6b7786097 100644
--- a/drivers/gpu/drm/cirrus/cirrus_ttm.c
+++ b/drivers/gpu/drm/cirrus/cirrus_ttm.c
@@ -259,7 +259,9 @@ int cirrus_mm_init(struct cirrus_device *cirrus)
 
         ret = ttm_bo_device_init(&cirrus->ttm.bdev,
                                  cirrus->ttm.bo_global_ref.ref.object,
-                                 &cirrus_bo_driver, DRM_FILE_PAGE_OFFSET,
+                                 &cirrus_bo_driver,
+                                 dev->anon_inode->i_mapping,
+                                 DRM_FILE_PAGE_OFFSET,
                                  true);
         if (ret) {
                 DRM_ERROR("Error initialising bo driver; %d\n", ret);
@@ -329,7 +331,6 @@ int cirrus_bo_create(struct drm_device *dev, int size, int align,
         }
 
         cirrusbo->bo.bdev = &cirrus->ttm.bdev;
-        cirrusbo->bo.bdev->dev_mapping = dev->dev_mapping;
 
         cirrus_ttm_placement(cirrusbo, TTM_PL_FLAG_VRAM | TTM_PL_FLAG_SYSTEM);
 
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index 3b7d32da1604..d8b7099abece 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -38,12 +38,15 @@
 #include <drm/drm_edid.h>
 #include <drm/drm_fourcc.h>
 
+#include "drm_crtc_internal.h"
+
 /**
  * drm_modeset_lock_all - take all modeset locks
  * @dev: drm device
  *
  * This function takes all modeset locks, suitable where a more fine-grained
- * scheme isn't (yet) implemented.
+ * scheme isn't (yet) implemented. Locks must be dropped with
+ * drm_modeset_unlock_all.
  */
 void drm_modeset_lock_all(struct drm_device *dev)
 {
@@ -59,6 +62,8 @@ EXPORT_SYMBOL(drm_modeset_lock_all);
 /**
  * drm_modeset_unlock_all - drop all modeset locks
  * @dev: device
+ *
+ * This function drop all modeset locks taken by drm_modeset_lock_all.
  */
 void drm_modeset_unlock_all(struct drm_device *dev)
 {
@@ -74,6 +79,8 @@ EXPORT_SYMBOL(drm_modeset_unlock_all);
 /**
  * drm_warn_on_modeset_not_all_locked - check that all modeset locks are locked
  * @dev: device
+ *
+ * Useful as a debug assert.
  */
 void drm_warn_on_modeset_not_all_locked(struct drm_device *dev)
 {
@@ -114,6 +121,13 @@ static const struct drm_prop_enum_list drm_dpms_enum_list[] =
 
 DRM_ENUM_NAME_FN(drm_get_dpms_name, drm_dpms_enum_list)
 
+static const struct drm_prop_enum_list drm_plane_type_enum_list[] =
+{
+        { DRM_PLANE_TYPE_OVERLAY, "Overlay" },
+        { DRM_PLANE_TYPE_PRIMARY, "Primary" },
+        { DRM_PLANE_TYPE_CURSOR, "Cursor" },
+};
+
 /*
  * Optional properties
  */
@@ -215,6 +229,16 @@ static const struct drm_prop_enum_list drm_encoder_enum_list[] =
         { DRM_MODE_ENCODER_DSI, "DSI" },
 };
 
+static const struct drm_prop_enum_list drm_subpixel_enum_list[] =
+{
+        { SubPixelUnknown, "Unknown" },
+        { SubPixelHorizontalRGB, "Horizontal RGB" },
+        { SubPixelHorizontalBGR, "Horizontal BGR" },
+        { SubPixelVerticalRGB, "Vertical RGB" },
+        { SubPixelVerticalBGR, "Vertical BGR" },
+        { SubPixelNone, "None" },
+};
+
 void drm_connector_ida_init(void)
 {
         int i;
@@ -231,6 +255,15 @@ void drm_connector_ida_destroy(void)
                 ida_destroy(&drm_connector_enum_list[i].ida);
 }
 
+/**
+ * drm_get_encoder_name - return a string for encoder
+ * @encoder: encoder to compute name of
+ *
+ * Note that the buffer used by this function is globally shared and owned by
+ * the function itself.
+ *
+ * FIXME: This isn't really multithreading safe.
+ */
 const char *drm_get_encoder_name(const struct drm_encoder *encoder)
 {
         static char buf[32];
@@ -242,6 +275,15 @@ const char *drm_get_encoder_name(const struct drm_encoder *encoder)
 }
 EXPORT_SYMBOL(drm_get_encoder_name);
 
+/**
+ * drm_get_connector_name - return a string for connector
+ * @connector: connector to compute name of
+ *
+ * Note that the buffer used by this function is globally shared and owned by
+ * the function itself.
+ *
+ * FIXME: This isn't really multithreading safe.
+ */
 const char *drm_get_connector_name(const struct drm_connector *connector)
 {
         static char buf[32];
@@ -253,6 +295,13 @@ const char *drm_get_connector_name(const struct drm_connector *connector)
 }
 EXPORT_SYMBOL(drm_get_connector_name);
 
+/**
+ * drm_get_connector_status_name - return a string for connector status
+ * @status: connector status to compute name of
+ *
+ * In contrast to the other drm_get_*_name functions this one here returns a
+ * const pointer and hence is threadsafe.
+ */
 const char *drm_get_connector_status_name(enum drm_connector_status status)
 {
         if (status == connector_status_connected)
@@ -264,11 +313,33 @@ const char *drm_get_connector_status_name(enum drm_connector_status status)
 }
 EXPORT_SYMBOL(drm_get_connector_status_name);
 
+/**
+ * drm_get_subpixel_order_name - return a string for a given subpixel enum
+ * @order: enum of subpixel_order
+ *
+ * Note you could abuse this and return something out of bounds, but that
+ * would be a caller error.  No unscrubbed user data should make it here.
+ */
+const char *drm_get_subpixel_order_name(enum subpixel_order order)
+{
+        return drm_subpixel_enum_list[order].name;
+}
+EXPORT_SYMBOL(drm_get_subpixel_order_name);
+
 static char printable_char(int c)
 {
         return isascii(c) && isprint(c) ? c : '?';
 }
 
+/**
+ * drm_get_format_name - return a string for drm fourcc format
+ * @format: format to compute name of
+ *
+ * Note that the buffer used by this function is globally shared and owned by
+ * the function itself.
+ *
+ * FIXME: This isn't really multithreading safe.
+ */
 const char *drm_get_format_name(uint32_t format)
 {
         static char buf[32];
@@ -293,14 +364,16 @@ EXPORT_SYMBOL(drm_get_format_name);
  * @obj_type: object type
  *
  * Create a unique identifier based on @ptr in @dev's identifier space.  Used
- * for tracking modes, CRTCs and connectors.
+ * for tracking modes, CRTCs and connectors. Note that despite the _get postfix
+ * modeset identifiers are _not_ reference counted. Hence don't use this for
+ * reference counted modeset objects like framebuffers.
  *
- * RETURNS:
+ * Returns:
  * New unique (relative to other objects in @dev) integer identifier for the
  * object.
  */
-static int drm_mode_object_get(struct drm_device *dev,
-                               struct drm_mode_object *obj, uint32_t obj_type)
+int drm_mode_object_get(struct drm_device *dev,
+                        struct drm_mode_object *obj, uint32_t obj_type)
 {
         int ret;
 
@@ -324,10 +397,12 @@ static int drm_mode_object_get(struct drm_device *dev,
  * @dev: DRM device
  * @object: object to free
  *
- * Free @id from @dev's unique identifier pool.
+ * Free @id from @dev's unique identifier pool. Note that despite the _get
+ * postfix modeset identifiers are _not_ reference counted. Hence don't use this
+ * for reference counted modeset objects like framebuffers.
  */
-static void drm_mode_object_put(struct drm_device *dev,
-                                struct drm_mode_object *object)
+void drm_mode_object_put(struct drm_device *dev,
+                         struct drm_mode_object *object)
 {
         mutex_lock(&dev->mode_config.idr_mutex);
         idr_remove(&dev->mode_config.crtc_idr, object->id);
@@ -377,7 +452,7 @@ EXPORT_SYMBOL(drm_mode_object_find);
  * since all the fb attributes are invariant over its lifetime, no further
  * locking but only correct reference counting is required.
  *
- * RETURNS:
+ * Returns:
  * Zero on success, error code on failure.
  */
 int drm_framebuffer_init(struct drm_device *dev, struct drm_framebuffer *fb,
@@ -438,7 +513,7 @@ static struct drm_framebuffer *__drm_framebuffer_lookup(struct drm_device *dev,
  *
  * If successful, this grabs an additional reference to the framebuffer -
  * callers need to make sure to eventually unreference the returned framebuffer
- * again.
+ * again, using @drm_framebuffer_unreference.
  */
 struct drm_framebuffer *drm_framebuffer_lookup(struct drm_device *dev,
                                                uint32_t id)
@@ -471,6 +546,8 @@ EXPORT_SYMBOL(drm_framebuffer_unreference);
 /**
  * drm_framebuffer_reference - incr the fb refcnt
  * @fb: framebuffer
+ *
+ * This functions increments the fb's refcount.
  */
 void drm_framebuffer_reference(struct drm_framebuffer *fb)
 {
@@ -527,8 +604,9 @@ EXPORT_SYMBOL(drm_framebuffer_unregister_private);
  * drm_framebuffer_cleanup - remove a framebuffer object
  * @fb: framebuffer to remove
  *
- * Cleanup references to a user-created framebuffer. This function is intended
- * to be used from the drivers ->destroy callback.
+ * Cleanup framebuffer. This function is intended to be used from the drivers
+ * ->destroy callback. It can also be used to clean up driver private
+ *  framebuffers embedded into a larger structure.
  *
  * Note that this function does not remove the fb from active usuage - if it is
  * still used anywhere, hilarity can ensue since userspace could call getfb on
@@ -591,7 +669,7 @@ void drm_framebuffer_remove(struct drm_framebuffer *fb)
                 drm_modeset_lock_all(dev);
                 /* remove from any CRTC */
                 list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-                        if (crtc->fb == fb) {
+                        if (crtc->primary->fb == fb) {
                                 /* should turn off the crtc */
                                 memset(&set, 0, sizeof(struct drm_mode_set));
                                 set.crtc = crtc;
@@ -614,18 +692,23 @@ void drm_framebuffer_remove(struct drm_framebuffer *fb)
 EXPORT_SYMBOL(drm_framebuffer_remove);
 
 /**
- * drm_crtc_init - Initialise a new CRTC object
+ * drm_crtc_init_with_planes - Initialise a new CRTC object with
+ *    specified primary and cursor planes.
  * @dev: DRM device
  * @crtc: CRTC object to init
+ * @primary: Primary plane for CRTC
+ * @cursor: Cursor plane for CRTC
  * @funcs: callbacks for the new CRTC
  *
  * Inits a new object created as base part of a driver crtc object.
  *
- * RETURNS:
+ * Returns:
  * Zero on success, error code on failure.
  */
-int drm_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
-                   const struct drm_crtc_funcs *funcs)
+int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc,
+                              struct drm_plane *primary,
+                              void *cursor,
+                              const struct drm_crtc_funcs *funcs)
 {
         int ret;
 
@@ -646,12 +729,16 @@ int drm_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
         list_add_tail(&crtc->head, &dev->mode_config.crtc_list);
         dev->mode_config.num_crtc++;
 
+        crtc->primary = primary;
+        if (primary)
+                primary->possible_crtcs = 1 << drm_crtc_index(crtc);
+
  out:
         drm_modeset_unlock_all(dev);
 
         return ret;
 }
-EXPORT_SYMBOL(drm_crtc_init);
+EXPORT_SYMBOL(drm_crtc_init_with_planes);
 
 /**
  * drm_crtc_cleanup - Clean up the core crtc usage
@@ -697,20 +784,6 @@ unsigned int drm_crtc_index(struct drm_crtc *crtc)
 }
 EXPORT_SYMBOL(drm_crtc_index);
 
-/**
- * drm_mode_probed_add - add a mode to a connector's probed mode list
- * @connector: connector the new mode
- * @mode: mode data
- *
- * Add @mode to @connector's mode list for later use.
- */
-void drm_mode_probed_add(struct drm_connector *connector,
-                         struct drm_display_mode *mode)
-{
-        list_add_tail(&mode->head, &connector->probed_modes);
-}
-EXPORT_SYMBOL(drm_mode_probed_add);
-
 /*
  * drm_mode_remove - remove and free a mode
  * @connector: connector list to modify
@@ -735,7 +808,7 @@ static void drm_mode_remove(struct drm_connector *connector,
  * Initialises a preallocated connector. Connectors should be
  * subclassed as part of driver connector objects.
  *
- * RETURNS:
+ * Returns:
  * Zero on success, error code on failure.
  */
 int drm_connector_init(struct drm_device *dev,
@@ -813,6 +886,14 @@ void drm_connector_cleanup(struct drm_connector *connector)
 }
 EXPORT_SYMBOL(drm_connector_cleanup);
 
+/**
+ * drm_connector_unplug_all - unregister connector userspace interfaces
+ * @dev: drm device
+ *
+ * This function unregisters all connector userspace interfaces in sysfs. Should
+ * be call when the device is disconnected, e.g. from an usb driver's
+ * ->disconnect callback.
+ */
 void drm_connector_unplug_all(struct drm_device *dev)
 {
         struct drm_connector *connector;
@@ -824,6 +905,18 @@ void drm_connector_unplug_all(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_connector_unplug_all);
 
+/**
+ * drm_bridge_init - initialize a drm transcoder/bridge
+ * @dev: drm device
+ * @bridge: transcoder/bridge to set up
+ * @funcs: bridge function table
+ *
+ * Initialises a preallocated bridge. Bridges should be
+ * subclassed as part of driver connector objects.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
 int drm_bridge_init(struct drm_device *dev, struct drm_bridge *bridge,
                 const struct drm_bridge_funcs *funcs)
 {
@@ -847,6 +940,12 @@ int drm_bridge_init(struct drm_device *dev, struct drm_bridge *bridge,
 }
 EXPORT_SYMBOL(drm_bridge_init);
 
+/**
+ * drm_bridge_cleanup - cleans up an initialised bridge
+ * @bridge: bridge to cleanup
+ *
+ * Cleans up the bridge but doesn't free the object.
+ */
 void drm_bridge_cleanup(struct drm_bridge *bridge)
 {
         struct drm_device *dev = bridge->dev;
@@ -859,6 +958,19 @@ void drm_bridge_cleanup(struct drm_bridge *bridge)
 }
 EXPORT_SYMBOL(drm_bridge_cleanup);
 
+/**
+ * drm_encoder_init - Init a preallocated encoder
+ * @dev: drm device
+ * @encoder: the encoder to init
+ * @funcs: callbacks for this encoder
+ * @encoder_type: user visible type of the encoder
+ *
+ * Initialises a preallocated encoder. Encoder should be
+ * subclassed as part of driver encoder objects.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
 int drm_encoder_init(struct drm_device *dev,
                       struct drm_encoder *encoder,
                       const struct drm_encoder_funcs *funcs,
@@ -886,6 +998,12 @@ int drm_encoder_init(struct drm_device *dev,
 }
 EXPORT_SYMBOL(drm_encoder_init);
 
+/**
+ * drm_encoder_cleanup - cleans up an initialised encoder
+ * @encoder: encoder to cleanup
+ *
+ * Cleans up the encoder but doesn't free the object.
+ */
 void drm_encoder_cleanup(struct drm_encoder *encoder)
 {
         struct drm_device *dev = encoder->dev;
@@ -898,25 +1016,25 @@ void drm_encoder_cleanup(struct drm_encoder *encoder)
 EXPORT_SYMBOL(drm_encoder_cleanup);
 
 /**
- * drm_plane_init - Initialise a new plane object
+ * drm_universal_plane_init - Initialize a new universal plane object
  * @dev: DRM device
  * @plane: plane object to init
  * @possible_crtcs: bitmask of possible CRTCs
  * @funcs: callbacks for the new plane
  * @formats: array of supported formats (%DRM_FORMAT_*)
  * @format_count: number of elements in @formats
- * @priv: plane is private (hidden from userspace)?
+ * @type: type of plane (overlay, primary, cursor)
  *
- * Inits a new object created as base part of a driver plane object.
+ * Initializes a plane object of type @type.
  *
- * RETURNS:
+ * Returns:
  * Zero on success, error code on failure.
  */
-int drm_plane_init(struct drm_device *dev, struct drm_plane *plane,
-                   unsigned long possible_crtcs,
-                   const struct drm_plane_funcs *funcs,
-                   const uint32_t *formats, uint32_t format_count,
-                   bool priv)
+int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane,
+                             unsigned long possible_crtcs,
+                             const struct drm_plane_funcs *funcs,
+                             const uint32_t *formats, uint32_t format_count,
+                             enum drm_plane_type type)
 {
         int ret;
 
@@ -941,23 +1059,53 @@ int drm_plane_init(struct drm_device *dev, struct drm_plane *plane,
         memcpy(plane->format_types, formats, format_count * sizeof(uint32_t));
         plane->format_count = format_count;
         plane->possible_crtcs = possible_crtcs;
+        plane->type = type;
 
-        /* private planes are not exposed to userspace, but depending on
-         * display hardware, might be convenient to allow sharing programming
-         * for the scanout engine with the crtc implementation.
-         */
-        if (!priv) {
-                list_add_tail(&plane->head, &dev->mode_config.plane_list);
-                dev->mode_config.num_plane++;
-        } else {
-                INIT_LIST_HEAD(&plane->head);
-        }
+        list_add_tail(&plane->head, &dev->mode_config.plane_list);
+        dev->mode_config.num_total_plane++;
+        if (plane->type == DRM_PLANE_TYPE_OVERLAY)
+                dev->mode_config.num_overlay_plane++;
+
+        drm_object_attach_property(&plane->base,
+                                   dev->mode_config.plane_type_property,
+                                   plane->type);
 
  out:
         drm_modeset_unlock_all(dev);
 
         return ret;
 }
+EXPORT_SYMBOL(drm_universal_plane_init);
+
+/**
+ * drm_plane_init - Initialize a legacy plane
+ * @dev: DRM device
+ * @plane: plane object to init
+ * @possible_crtcs: bitmask of possible CRTCs
+ * @funcs: callbacks for the new plane
+ * @formats: array of supported formats (%DRM_FORMAT_*)
+ * @format_count: number of elements in @formats
+ * @is_primary: plane type (primary vs overlay)
+ *
+ * Legacy API to initialize a DRM plane.
+ *
+ * New drivers should call drm_universal_plane_init() instead.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_plane_init(struct drm_device *dev, struct drm_plane *plane,
+                   unsigned long possible_crtcs,
+                   const struct drm_plane_funcs *funcs,
+                   const uint32_t *formats, uint32_t format_count,
+                   bool is_primary)
+{
+        enum drm_plane_type type;
+
+        type = is_primary ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
+        return drm_universal_plane_init(dev, plane, possible_crtcs, funcs,
+                                        formats, format_count, type);
+}
 EXPORT_SYMBOL(drm_plane_init);
 
 /**
@@ -975,11 +1123,13 @@ void drm_plane_cleanup(struct drm_plane *plane)
         drm_modeset_lock_all(dev);
         kfree(plane->format_types);
         drm_mode_object_put(dev, &plane->base);
-        /* if not added to a list, it must be a private plane */
-        if (!list_empty(&plane->head)) {
-                list_del(&plane->head);
-                dev->mode_config.num_plane--;
-        }
+
+        BUG_ON(list_empty(&plane->head));
+
+        list_del(&plane->head);
+        dev->mode_config.num_total_plane--;
+        if (plane->type == DRM_PLANE_TYPE_OVERLAY)
+                dev->mode_config.num_overlay_plane--;
         drm_modeset_unlock_all(dev);
 }
 EXPORT_SYMBOL(drm_plane_cleanup);
@@ -1010,50 +1160,6 @@ void drm_plane_force_disable(struct drm_plane *plane)
 }
 EXPORT_SYMBOL(drm_plane_force_disable);
 
-/**
- * drm_mode_create - create a new display mode
- * @dev: DRM device
- *
- * Create a new drm_display_mode, give it an ID, and return it.
- *
- * RETURNS:
- * Pointer to new mode on success, NULL on error.
- */
-struct drm_display_mode *drm_mode_create(struct drm_device *dev)
-{
-        struct drm_display_mode *nmode;
-
-        nmode = kzalloc(sizeof(struct drm_display_mode), GFP_KERNEL);
-        if (!nmode)
-                return NULL;
-
-        if (drm_mode_object_get(dev, &nmode->base, DRM_MODE_OBJECT_MODE)) {
-                kfree(nmode);
-                return NULL;
-        }
-
-        return nmode;
-}
-EXPORT_SYMBOL(drm_mode_create);
-
-/**
- * drm_mode_destroy - remove a mode
- * @dev: DRM device
- * @mode: mode to remove
- *
- * Free @mode's unique identifier, then free it.
- */
-void drm_mode_destroy(struct drm_device *dev, struct drm_display_mode *mode)
-{
-        if (!mode)
-                return;
-
-        drm_mode_object_put(dev, &mode->base);
-
-        kfree(mode);
-}
-EXPORT_SYMBOL(drm_mode_destroy);
-
 static int drm_mode_create_standard_connector_properties(struct drm_device *dev)
 {
         struct drm_property *edid;
@@ -1075,6 +1181,21 @@ static int drm_mode_create_standard_connector_properties(struct drm_device *dev)
         return 0;
 }
 
+static int drm_mode_create_standard_plane_properties(struct drm_device *dev)
+{
+        struct drm_property *type;
+
+        /*
+         * Standard properties (apply to all planes)
+         */
+        type = drm_property_create_enum(dev, DRM_MODE_PROP_IMMUTABLE,
+                                        "type", drm_plane_type_enum_list,
+                                        ARRAY_SIZE(drm_plane_type_enum_list));
+        dev->mode_config.plane_type_property = type;
+
+        return 0;
+}
+
 /**
  * drm_mode_create_dvi_i_properties - create DVI-I specific connector properties
  * @dev: DRM device
@@ -1257,6 +1378,10 @@ static int drm_mode_group_init(struct drm_device *dev, struct drm_mode_group *gr
         return 0;
 }
 
+/*
+ * NOTE: Driver's shouldn't ever call drm_mode_group_init_legacy_group - it is
+ * the drm core's responsibility to set up mode control groups.
+ */
 int drm_mode_group_init_legacy_group(struct drm_device *dev,
                                      struct drm_mode_group *group)
 {
@@ -1333,7 +1458,7 @@ static void drm_crtc_convert_to_umode(struct drm_mode_modeinfo *out,
  * Convert a drm_mode_modeinfo into a drm_display_mode structure to return to
  * the caller.
  *
- * RETURNS:
+ * Returns:
  * Zero on success, errno on failure.
  */
 static int drm_crtc_convert_umode(struct drm_display_mode *out,
@@ -1376,7 +1501,7 @@ static int drm_crtc_convert_umode(struct drm_display_mode *out,
  *
  * Called by the user via ioctl.
  *
- * RETURNS:
+ * Returns:
  * Zero on success, errno on failure.
  */
 int drm_mode_getresources(struct drm_device *dev, void *data,
@@ -1429,9 +1554,9 @@ int drm_mode_getresources(struct drm_device *dev, void *data,
         mutex_unlock(&file_priv->fbs_lock);
 
         drm_modeset_lock_all(dev);
-        mode_group = &file_priv->master->minor->mode_group;
-        if (file_priv->master->minor->type == DRM_MINOR_CONTROL) {
+        if (!drm_is_primary_client(file_priv)) {
 
+                mode_group = NULL;
                 list_for_each(lh, &dev->mode_config.crtc_list)
                         crtc_count++;
 
@@ -1442,6 +1567,7 @@ int drm_mode_getresources(struct drm_device *dev, void *data,
                         encoder_count++;
         } else {
 
+                mode_group = &file_priv->master->minor->mode_group;
                 crtc_count = mode_group->num_crtcs;
                 connector_count = mode_group->num_connectors;
                 encoder_count = mode_group->num_encoders;
@@ -1456,7 +1582,7 @@ int drm_mode_getresources(struct drm_device *dev, void *data,
         if (card_res->count_crtcs >= crtc_count) {
                 copied = 0;
                 crtc_id = (uint32_t __user *)(unsigned long)card_res->crtc_id_ptr;
-                if (file_priv->master->minor->type == DRM_MINOR_CONTROL) {
+                if (!mode_group) {
                         list_for_each_entry(crtc, &dev->mode_config.crtc_list,
                                             head) {
                                 DRM_DEBUG_KMS("[CRTC:%d]\n", crtc->base.id);
@@ -1483,7 +1609,7 @@ int drm_mode_getresources(struct drm_device *dev, void *data,
         if (card_res->count_encoders >= encoder_count) {
                 copied = 0;
                 encoder_id = (uint32_t __user *)(unsigned long)card_res->encoder_id_ptr;
-                if (file_priv->master->minor->type == DRM_MINOR_CONTROL) {
+                if (!mode_group) {
                         list_for_each_entry(encoder,
                                             &dev->mode_config.encoder_list,
                                             head) {
@@ -1514,7 +1640,7 @@ int drm_mode_getresources(struct drm_device *dev, void *data,
         if (card_res->count_connectors >= connector_count) {
                 copied = 0;
                 connector_id = (uint32_t __user *)(unsigned long)card_res->connector_id_ptr;
-                if (file_priv->master->minor->type == DRM_MINOR_CONTROL) {
+                if (!mode_group) {
                         list_for_each_entry(connector,
                                             &dev->mode_config.connector_list,
                                             head) {
@@ -1561,7 +1687,7 @@ out:
  *
  * Called by the user via ioctl.
  *
- * RETURNS:
+ * Returns:
  * Zero on success, errno on failure.
  */
 int drm_mode_getcrtc(struct drm_device *dev,
@@ -1588,8 +1714,8 @@ int drm_mode_getcrtc(struct drm_device *dev,
         crtc_resp->x = crtc->x;
         crtc_resp->y = crtc->y;
         crtc_resp->gamma_size = crtc->gamma_size;
-        if (crtc->fb)
-                crtc_resp->fb_id = crtc->fb->base.id;
+        if (crtc->primary->fb)
+                crtc_resp->fb_id = crtc->primary->fb->base.id;
         else
                 crtc_resp->fb_id = 0;
 
@@ -1630,7 +1756,7 @@ static bool drm_mode_expose_to_userspace(const struct drm_display_mode *mode,
  *
  * Called by the user via ioctl.
  *
- * RETURNS:
+ * Returns:
  * Zero on success, errno on failure.
  */
 int drm_mode_getconnector(struct drm_device *dev, void *data,
@@ -1765,6 +1891,19 @@ out:
         return ret;
 }
 
+/**
+ * drm_mode_getencoder - get encoder configuration
+ * @dev: drm device for the ioctl
+ * @data: data pointer for the ioctl
+ * @file_priv: drm file for the ioctl call
+ *
+ * Construct a encoder configuration structure to return to the user.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_getencoder(struct drm_device *dev, void *data,
                         struct drm_file *file_priv)
 {
@@ -1800,21 +1939,27 @@ out:
 }
 
 /**
- * drm_mode_getplane_res - get plane info
+ * drm_mode_getplane_res - enumerate all plane resources
  * @dev: DRM device
  * @data: ioctl data
  * @file_priv: DRM file info
  *
- * Return an plane count and set of IDs.
+ * Construct a list of plane ids to return to the user.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
  */
 int drm_mode_getplane_res(struct drm_device *dev, void *data,
-                            struct drm_file *file_priv)
+                          struct drm_file *file_priv)
 {
         struct drm_mode_get_plane_res *plane_resp = data;
         struct drm_mode_config *config;
         struct drm_plane *plane;
         uint32_t __user *plane_ptr;
         int copied = 0, ret = 0;
+        unsigned num_planes;
 
         if (!drm_core_check_feature(dev, DRIVER_MODESET))
                 return -EINVAL;
@@ -1822,15 +1967,28 @@ int drm_mode_getplane_res(struct drm_device *dev, void *data,
         drm_modeset_lock_all(dev);
         config = &dev->mode_config;
 
+        if (file_priv->universal_planes)
+                num_planes = config->num_total_plane;
+        else
+                num_planes = config->num_overlay_plane;
+
         /*
          * This ioctl is called twice, once to determine how much space is
          * needed, and the 2nd time to fill it.
          */
-        if (config->num_plane &&
-            (plane_resp->count_planes >= config->num_plane)) {
+        if (num_planes &&
+            (plane_resp->count_planes >= num_planes)) {
                 plane_ptr = (uint32_t __user *)(unsigned long)plane_resp->plane_id_ptr;
 
                 list_for_each_entry(plane, &config->plane_list, head) {
+                        /*
+                         * Unless userspace set the 'universal planes'
+                         * capability bit, only advertise overlays.
+                         */
+                        if (plane->type != DRM_PLANE_TYPE_OVERLAY &&
+                            !file_priv->universal_planes)
+                                continue;
+
                         if (put_user(plane->base.id, plane_ptr + copied)) {
                                 ret = -EFAULT;
                                 goto out;
@@ -1838,7 +1996,7 @@ int drm_mode_getplane_res(struct drm_device *dev, void *data,
                         copied++;
                 }
         }
-        plane_resp->count_planes = config->num_plane;
+        plane_resp->count_planes = num_planes;
 
 out:
         drm_modeset_unlock_all(dev);
@@ -1846,16 +2004,20 @@ out:
 }
 
 /**
- * drm_mode_getplane - get plane info
+ * drm_mode_getplane - get plane configuration
  * @dev: DRM device
  * @data: ioctl data
  * @file_priv: DRM file info
  *
- * Return plane info, including formats supported, gamma size, any
- * current fb, etc.
+ * Construct a plane configuration structure to return to the user.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
  */
 int drm_mode_getplane(struct drm_device *dev, void *data,
-                        struct drm_file *file_priv)
+                      struct drm_file *file_priv)
 {
         struct drm_mode_get_plane *plane_resp = data;
         struct drm_mode_object *obj;
@@ -1911,16 +2073,19 @@ out:
 }
 
 /**
- * drm_mode_setplane - set up or tear down an plane
+ * drm_mode_setplane - configure a plane's configuration
  * @dev: DRM device
  * @data: ioctl data*
  * @file_priv: DRM file info
  *
- * Set plane info, including placement, fb, scaling, and other factors.
+ * Set plane configuration, including placement, fb, scaling, and other factors.
  * Or pass a NULL fb to disable.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
  */
 int drm_mode_setplane(struct drm_device *dev, void *data,
-                        struct drm_file *file_priv)
+                      struct drm_file *file_priv)
 {
         struct drm_mode_set_plane *plane_req = data;
         struct drm_mode_object *obj;
@@ -2050,6 +2215,9 @@ out:
  *
  * This is a little helper to wrap internal calls to the ->set_config driver
  * interface. The only thing it adds is correct refcounting dance.
+ * 
+ * Returns:
+ * Zero on success, errno on failure.
  */
 int drm_mode_set_config_internal(struct drm_mode_set *set)
 {
@@ -2064,19 +2232,21 @@ int drm_mode_set_config_internal(struct drm_mode_set *set)
          * crtcs. Atomic modeset will have saner semantics ...
          */
         list_for_each_entry(tmp, &crtc->dev->mode_config.crtc_list, head)
-                tmp->old_fb = tmp->fb;
+                tmp->old_fb = tmp->primary->fb;
 
         fb = set->fb;
 
         ret = crtc->funcs->set_config(set);
         if (ret == 0) {
+                crtc->primary->crtc = crtc;
+
                 /* crtc->fb must be updated by ->set_config, enforces this. */
-                WARN_ON(fb != crtc->fb);
+                WARN_ON(fb != crtc->primary->fb);
         }
 
         list_for_each_entry(tmp, &crtc->dev->mode_config.crtc_list, head) {
-                if (tmp->fb)
-                        drm_framebuffer_reference(tmp->fb);
+                if (tmp->primary->fb)
+                        drm_framebuffer_reference(tmp->primary->fb);
                 if (tmp->old_fb)
                         drm_framebuffer_unreference(tmp->old_fb);
         }
@@ -2085,14 +2255,19 @@ int drm_mode_set_config_internal(struct drm_mode_set *set)
 }
 EXPORT_SYMBOL(drm_mode_set_config_internal);
 
-/*
- * Checks that the framebuffer is big enough for the CRTC viewport
- * (x, y, hdisplay, vdisplay)
+/**
+ * drm_crtc_check_viewport - Checks that a framebuffer is big enough for the
+ *     CRTC viewport
+ * @crtc: CRTC that framebuffer will be displayed on
+ * @x: x panning
+ * @y: y panning
+ * @mode: mode that framebuffer will be displayed under
+ * @fb: framebuffer to check size of
  */
-static int drm_crtc_check_viewport(const struct drm_crtc *crtc,
-                                   int x, int y,
-                                   const struct drm_display_mode *mode,
-                                   const struct drm_framebuffer *fb)
+int drm_crtc_check_viewport(const struct drm_crtc *crtc,
+                            int x, int y,
+                            const struct drm_display_mode *mode,
+                            const struct drm_framebuffer *fb)
 
 {
         int hdisplay, vdisplay;
@@ -2123,6 +2298,7 @@ static int drm_crtc_check_viewport(const struct drm_crtc *crtc,
 
         return 0;
 }
+EXPORT_SYMBOL(drm_crtc_check_viewport);
 
 /**
  * drm_mode_setcrtc - set CRTC configuration
@@ -2134,7 +2310,7 @@ static int drm_crtc_check_viewport(const struct drm_crtc *crtc,
  *
  * Called by the user via ioctl.
  *
- * RETURNS:
+ * Returns:
  * Zero on success, errno on failure.
  */
 int drm_mode_setcrtc(struct drm_device *dev, void *data,
@@ -2174,12 +2350,12 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
                 /* If we have a mode we need a framebuffer. */
                 /* If we pass -1, set the mode with the currently bound fb */
                 if (crtc_req->fb_id == -1) {
-                        if (!crtc->fb) {
+                        if (!crtc->primary->fb) {
                                 DRM_DEBUG_KMS("CRTC doesn't have current FB\n");
                                 ret = -EINVAL;
                                 goto out;
                         }
-                        fb = crtc->fb;
+                        fb = crtc->primary->fb;
                         /* Make refcounting symmetric with the lookup path. */
                         drm_framebuffer_reference(fb);
                 } else {
@@ -2336,8 +2512,23 @@ out:
         return ret;
 
 }
+
+
+/**
+ * drm_mode_cursor_ioctl - set CRTC's cursor configuration
+ * @dev: drm device for the ioctl
+ * @data: data pointer for the ioctl
+ * @file_priv: drm file for the ioctl call
+ *
+ * Set the cursor configuration based on user request.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_cursor_ioctl(struct drm_device *dev,
-                        void *data, struct drm_file *file_priv)
+                          void *data, struct drm_file *file_priv)
 {
         struct drm_mode_cursor *req = data;
         struct drm_mode_cursor2 new_req;
@@ -2348,6 +2539,21 @@ int drm_mode_cursor_ioctl(struct drm_device *dev,
         return drm_mode_cursor_common(dev, &new_req, file_priv);
 }
 
+/**
+ * drm_mode_cursor2_ioctl - set CRTC's cursor configuration
+ * @dev: drm device for the ioctl
+ * @data: data pointer for the ioctl
+ * @file_priv: drm file for the ioctl call
+ *
+ * Set the cursor configuration based on user request. This implements the 2nd
+ * version of the cursor ioctl, which allows userspace to additionally specify
+ * the hotspot of the pointer.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_cursor2_ioctl(struct drm_device *dev,
                            void *data, struct drm_file *file_priv)
 {
@@ -2355,7 +2561,14 @@ int drm_mode_cursor2_ioctl(struct drm_device *dev,
         return drm_mode_cursor_common(dev, req, file_priv);
 }
 
-/* Original addfb only supported RGB formats, so figure out which one */
+/**
+ * drm_mode_legacy_fb_format - compute drm fourcc code from legacy description
+ * @bpp: bits per pixels
+ * @depth: bit depth per pixel
+ *
+ * Computes a drm fourcc pixel format code for the given @bpp/@depth values.
+ * Useful in fbdev emulation code, since that deals in those values.
+ */
 uint32_t drm_mode_legacy_fb_format(uint32_t bpp, uint32_t depth)
 {
         uint32_t fmt;
@@ -2397,11 +2610,12 @@ EXPORT_SYMBOL(drm_mode_legacy_fb_format);
  * @data: data pointer for the ioctl
  * @file_priv: drm file for the ioctl call
  *
- * Add a new FB to the specified CRTC, given a user request.
+ * Add a new FB to the specified CRTC, given a user request. This is the
+ * original addfb ioclt which only supported RGB formats.
  *
  * Called by the user via ioctl.
  *
- * RETURNS:
+ * Returns:
  * Zero on success, errno on failure.
  */
 int drm_mode_addfb(struct drm_device *dev,
@@ -2574,11 +2788,13 @@ static int framebuffer_check(const struct drm_mode_fb_cmd2 *r)
  * @data: data pointer for the ioctl
  * @file_priv: drm file for the ioctl call
  *
- * Add a new FB to the specified CRTC, given a user request with format.
+ * Add a new FB to the specified CRTC, given a user request with format. This is
+ * the 2nd version of the addfb ioctl, which supports multi-planar framebuffers
+ * and uses fourcc codes as pixel format specifiers.
  *
  * Called by the user via ioctl.
  *
- * RETURNS:
+ * Returns:
  * Zero on success, errno on failure.
  */
 int drm_mode_addfb2(struct drm_device *dev,
@@ -2638,7 +2854,7 @@ int drm_mode_addfb2(struct drm_device *dev,
  *
  * Called by the user via ioctl.
  *
- * RETURNS:
+ * Returns:
  * Zero on success, errno on failure.
  */
 int drm_mode_rmfb(struct drm_device *dev,
@@ -2692,7 +2908,7 @@ fail_lookup:
  *
  * Called by the user via ioctl.
  *
- * RETURNS:
+ * Returns:
  * Zero on success, errno on failure.
  */
 int drm_mode_getfb(struct drm_device *dev,
@@ -2715,7 +2931,8 @@ int drm_mode_getfb(struct drm_device *dev,
         r->bpp = fb->bits_per_pixel;
         r->pitch = fb->pitches[0];
         if (fb->funcs->create_handle) {
-                if (file_priv->is_master || capable(CAP_SYS_ADMIN)) {
+                if (file_priv->is_master || capable(CAP_SYS_ADMIN) ||
+                    drm_is_control_client(file_priv)) {
                         ret = fb->funcs->create_handle(fb, file_priv,
                                                        &r->handle);
                 } else {
@@ -2736,6 +2953,25 @@ int drm_mode_getfb(struct drm_device *dev,
         return ret;
 }
 
+/**
+ * drm_mode_dirtyfb_ioctl - flush frontbuffer rendering on an FB
+ * @dev: drm device for the ioctl
+ * @data: data pointer for the ioctl
+ * @file_priv: drm file for the ioctl call
+ *
+ * Lookup the FB and flush out the damaged area supplied by userspace as a clip
+ * rectangle list. Generic userspace which does frontbuffer rendering must call
+ * this ioctl to flush out the changes on manual-update display outputs, e.g.
+ * usb display-link, mipi manual update panels or edp panel self refresh modes.
+ *
+ * Modesetting drivers which always update the frontbuffer do not need to
+ * implement the corresponding ->dirty framebuffer callback.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_dirtyfb_ioctl(struct drm_device *dev,
                            void *data, struct drm_file *file_priv)
 {
@@ -2813,7 +3049,7 @@ out_err1:
  *
  * Called by the user via ioctl.
  *
- * RETURNS:
+ * Returns:
  * Zero on success, errno on failure.
  */
 void drm_fb_release(struct drm_file *priv)
@@ -2837,6 +3073,20 @@ void drm_fb_release(struct drm_file *priv)
         mutex_unlock(&priv->fbs_lock);
 }
 
+/**
+ * drm_property_create - create a new property type
+ * @dev: drm device
+ * @flags: flags specifying the property type
+ * @name: name of the property
+ * @num_values: number of pre-defined values
+ *
+ * This creates a new generic drm property which can then be attached to a drm
+ * object with drm_object_attach_property. The returned property object must be
+ * freed with drm_property_destroy.
+ *
+ * Returns:
+ * A pointer to the newly created property on success, NULL on failure.
+ */
 struct drm_property *drm_property_create(struct drm_device *dev, int flags,
                                          const char *name, int num_values)
 {
@@ -2875,6 +3125,24 @@ fail:
 }
 EXPORT_SYMBOL(drm_property_create);
 
+/**
+ * drm_property_create - create a new enumeration property type
+ * @dev: drm device
+ * @flags: flags specifying the property type
+ * @name: name of the property
+ * @props: enumeration lists with property values
+ * @num_values: number of pre-defined values
+ *
+ * This creates a new generic drm property which can then be attached to a drm
+ * object with drm_object_attach_property. The returned property object must be
+ * freed with drm_property_destroy.
+ *
+ * Userspace is only allowed to set one of the predefined values for enumeration
+ * properties.
+ *
+ * Returns:
+ * A pointer to the newly created property on success, NULL on failure.
+ */
 struct drm_property *drm_property_create_enum(struct drm_device *dev, int flags,
                                          const char *name,
                                          const struct drm_prop_enum_list *props,
@@ -2903,6 +3171,24 @@ struct drm_property *drm_property_create_enum(struct drm_device *dev, int flags,
 }
 EXPORT_SYMBOL(drm_property_create_enum);
 
+/**
+ * drm_property_create - create a new bitmask property type
+ * @dev: drm device
+ * @flags: flags specifying the property type
+ * @name: name of the property
+ * @props: enumeration lists with property bitflags
+ * @num_values: number of pre-defined values
+ *
+ * This creates a new generic drm property which can then be attached to a drm
+ * object with drm_object_attach_property. The returned property object must be
+ * freed with drm_property_destroy.
+ *
+ * Compared to plain enumeration properties userspace is allowed to set any
+ * or'ed together combination of the predefined property bitflag values
+ *
+ * Returns:
+ * A pointer to the newly created property on success, NULL on failure.
+ */
 struct drm_property *drm_property_create_bitmask(struct drm_device *dev,
                                          int flags, const char *name,
                                          const struct drm_prop_enum_list *props,
@@ -2931,6 +3217,24 @@ struct drm_property *drm_property_create_bitmask(struct drm_device *dev,
 }
 EXPORT_SYMBOL(drm_property_create_bitmask);
 
+/**
+ * drm_property_create - create a new ranged property type
+ * @dev: drm device
+ * @flags: flags specifying the property type
+ * @name: name of the property
+ * @min: minimum value of the property
+ * @max: maximum value of the property
+ *
+ * This creates a new generic drm property which can then be attached to a drm
+ * object with drm_object_attach_property. The returned property object must be
+ * freed with drm_property_destroy.
+ *
+ * Userspace is allowed to set any interger value in the (min, max) range
+ * inclusive.
+ *
+ * Returns:
+ * A pointer to the newly created property on success, NULL on failure.
+ */
 struct drm_property *drm_property_create_range(struct drm_device *dev, int flags,
                                          const char *name,
                                          uint64_t min, uint64_t max)
@@ -2950,6 +3254,21 @@ struct drm_property *drm_property_create_range(struct drm_device *dev, int flags
 }
 EXPORT_SYMBOL(drm_property_create_range);
 
+/**
+ * drm_property_add_enum - add a possible value to an enumeration property
+ * @property: enumeration property to change
+ * @index: index of the new enumeration
+ * @value: value of the new enumeration
+ * @name: symbolic name of the new enumeration
+ *
+ * This functions adds enumerations to a property.
+ *
+ * It's use is deprecated, drivers should use one of the more specific helpers
+ * to directly create the property with all enumerations already attached.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
 int drm_property_add_enum(struct drm_property *property, int index,
                           uint64_t value, const char *name)
 {
@@ -2989,6 +3308,14 @@ int drm_property_add_enum(struct drm_property *property, int index,
 }
 EXPORT_SYMBOL(drm_property_add_enum);
 
+/**
+ * drm_property_destroy - destroy a drm property
+ * @dev: drm device
+ * @property: property to destry
+ *
+ * This function frees a property including any attached resources like
+ * enumeration values.
+ */
 void drm_property_destroy(struct drm_device *dev, struct drm_property *property)
 {
         struct drm_property_enum *prop_enum, *pt;
@@ -3006,6 +3333,16 @@ void drm_property_destroy(struct drm_device *dev, struct drm_property *property)
 }
 EXPORT_SYMBOL(drm_property_destroy);
 
+/**
+ * drm_object_attach_property - attach a property to a modeset object
+ * @obj: drm modeset object
+ * @property: property to attach
+ * @init_val: initial value of the property
+ *
+ * This attaches the given property to the modeset object with the given initial
+ * value. Currently this function cannot fail since the properties are stored in
+ * a statically sized array.
+ */
 void drm_object_attach_property(struct drm_mode_object *obj,
                                 struct drm_property *property,
                                 uint64_t init_val)
@@ -3026,6 +3363,19 @@ void drm_object_attach_property(struct drm_mode_object *obj,
 }
 EXPORT_SYMBOL(drm_object_attach_property);
 
+/**
+ * drm_object_property_set_value - set the value of a property
+ * @obj: drm mode object to set property value for
+ * @property: property to set
+ * @val: value the property should be set to
+ *
+ * This functions sets a given property on a given object. This function only
+ * changes the software state of the property, it does not call into the
+ * driver's ->set_property callback.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
 int drm_object_property_set_value(struct drm_mode_object *obj,
                                   struct drm_property *property, uint64_t val)
 {
@@ -3042,6 +3392,20 @@ int drm_object_property_set_value(struct drm_mode_object *obj,
 }
 EXPORT_SYMBOL(drm_object_property_set_value);
 
+/**
+ * drm_object_property_get_value - retrieve the value of a property
+ * @obj: drm mode object to get property value from
+ * @property: property to retrieve
+ * @val: storage for the property value
+ *
+ * This function retrieves the softare state of the given property for the given
+ * property. Since there is no driver callback to retrieve the current property
+ * value this might be out of sync with the hardware, depending upon the driver
+ * and property.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
 int drm_object_property_get_value(struct drm_mode_object *obj,
                                   struct drm_property *property, uint64_t *val)
 {
@@ -3058,6 +3422,19 @@ int drm_object_property_get_value(struct drm_mode_object *obj,
 }
 EXPORT_SYMBOL(drm_object_property_get_value);
 
+/**
+ * drm_mode_getproperty_ioctl - get the current value of a connector's property
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * This function retrieves the current value for an connectors's property.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_getproperty_ioctl(struct drm_device *dev,
                                void *data, struct drm_file *file_priv)
 {
@@ -3196,6 +3573,20 @@ static void drm_property_destroy_blob(struct drm_device *dev,
         kfree(blob);
 }
 
+/**
+ * drm_mode_getblob_ioctl - get the contents of a blob property value
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * This function retrieves the contents of a blob property. The value stored in
+ * an object's blob property is just a normal modeset object id.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_getblob_ioctl(struct drm_device *dev,
                            void *data, struct drm_file *file_priv)
 {
@@ -3230,6 +3621,17 @@ done:
         return ret;
 }
 
+/**
+ * drm_mode_connector_update_edid_property - update the edid property of a connector
+ * @connector: drm connector
+ * @edid: new value of the edid property
+ *
+ * This function creates a new blob modeset object and assigns its id to the
+ * connector's edid property.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_connector_update_edid_property(struct drm_connector *connector,
                                             struct edid *edid)
 {
@@ -3287,6 +3689,20 @@ static bool drm_property_change_is_valid(struct drm_property *property,
         }
 }
 
+/**
+ * drm_mode_connector_property_set_ioctl - set the current value of a connector property
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * This function sets the current value for a connectors's property. It also
+ * calls into a driver's ->set_property callback to update the hardware state
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_connector_property_set_ioctl(struct drm_device *dev,
                                        void *data, struct drm_file *file_priv)
 {
@@ -3353,6 +3769,21 @@ static int drm_mode_plane_set_obj_prop(struct drm_mode_object *obj,
         return ret;
 }
 
+/**
+ * drm_mode_getproperty_ioctl - get the current value of a object's property
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * This function retrieves the current value for an object's property. Compared
+ * to the connector specific ioctl this one is extended to also work on crtc and
+ * plane objects.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_obj_get_properties_ioctl(struct drm_device *dev, void *data,
                                       struct drm_file *file_priv)
 {
@@ -3409,6 +3840,22 @@ out:
         return ret;
 }
 
+/**
+ * drm_mode_obj_set_property_ioctl - set the current value of an object's property
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * This function sets the current value for an object's property. It also calls
+ * into a driver's ->set_property callback to update the hardware state.
+ * Compared to the connector specific ioctl this one is extended to also work on
+ * crtc and plane objects.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_obj_set_property_ioctl(struct drm_device *dev, void *data,
                                     struct drm_file *file_priv)
 {
@@ -3468,6 +3915,18 @@ out:
         return ret;
 }
 
+/**
+ * drm_mode_connector_attach_encoder - attach a connector to an encoder
+ * @connector: connector to attach
+ * @encoder: encoder to attach @connector to
+ *
+ * This function links up a connector to an encoder. Note that the routing
+ * restrictions between encoders and crtcs are exposed to userspace through the
+ * possible_clones and possible_crtcs bitmasks.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_connector_attach_encoder(struct drm_connector *connector,
                                       struct drm_encoder *encoder)
 {
@@ -3483,23 +3942,20 @@ int drm_mode_connector_attach_encoder(struct drm_connector *connector,
 }
 EXPORT_SYMBOL(drm_mode_connector_attach_encoder);
 
-void drm_mode_connector_detach_encoder(struct drm_connector *connector,
-                                    struct drm_encoder *encoder)
-{
-        int i;
-        for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
-                if (connector->encoder_ids[i] == encoder->base.id) {
-                        connector->encoder_ids[i] = 0;
-                        if (connector->encoder == encoder)
-                                connector->encoder = NULL;
-                        break;
-                }
-        }
-}
-EXPORT_SYMBOL(drm_mode_connector_detach_encoder);
-
+/**
+ * drm_mode_crtc_set_gamma_size - set the gamma table size
+ * @crtc: CRTC to set the gamma table size for
+ * @gamma_size: size of the gamma table
+ *
+ * Drivers which support gamma tables should set this to the supported gamma
+ * table size when initializing the CRTC. Currently the drm core only supports a
+ * fixed gamma table size.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_crtc_set_gamma_size(struct drm_crtc *crtc,
-                                  int gamma_size)
+                                 int gamma_size)
 {
         crtc->gamma_size = gamma_size;
 
@@ -3513,6 +3969,20 @@ int drm_mode_crtc_set_gamma_size(struct drm_crtc *crtc,
 }
 EXPORT_SYMBOL(drm_mode_crtc_set_gamma_size);
 
+/**
+ * drm_mode_gamma_set_ioctl - set the gamma table
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * Set the gamma table of a CRTC to the one passed in by the user. Userspace can
+ * inquire the required gamma table size through drm_mode_gamma_get_ioctl.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_gamma_set_ioctl(struct drm_device *dev,
                              void *data, struct drm_file *file_priv)
 {
@@ -3572,6 +4042,21 @@ out:
 
 }
 
+/**
+ * drm_mode_gamma_get_ioctl - get the gamma table
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * Copy the current gamma table into the storage provided. This also provides
+ * the gamma table size the driver expects, which can be used to size the
+ * allocated storage.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_gamma_get_ioctl(struct drm_device *dev,
                              void *data, struct drm_file *file_priv)
 {
@@ -3622,6 +4107,24 @@ out:
         return ret;
 }
 
+/**
+ * drm_mode_page_flip_ioctl - schedule an asynchronous fb update
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * This schedules an asynchronous update on a given CRTC, called page flip.
+ * Optionally a drm event is generated to signal the completion of the event.
+ * Generic drivers cannot assume that a pageflip with changed framebuffer
+ * properties (including driver specific metadata like tiling layout) will work,
+ * but some drivers support e.g. pixel format changes through the pageflip
+ * ioctl.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_page_flip_ioctl(struct drm_device *dev,
                              void *data, struct drm_file *file_priv)
 {
@@ -3646,7 +4149,7 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
         crtc = obj_to_crtc(obj);
 
         mutex_lock(&crtc->mutex);
-        if (crtc->fb == NULL) {
+        if (crtc->primary->fb == NULL) {
                 /* The framebuffer is currently unbound, presumably
                  * due to a hotplug event, that userspace has not
                  * yet discovered.
@@ -3668,7 +4171,7 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
         if (ret)
                 goto out;
 
-        if (crtc->fb->pixel_format != fb->pixel_format) {
+        if (crtc->primary->fb->pixel_format != fb->pixel_format) {
                 DRM_DEBUG_KMS("Page flip is not allowed to change frame buffer format.\n");
                 ret = -EINVAL;
                 goto out;
@@ -3701,7 +4204,7 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
                         (void (*) (struct drm_pending_event *)) kfree;
         }
 
-        old_fb = crtc->fb;
+        old_fb = crtc->primary->fb;
         ret = crtc->funcs->page_flip(crtc, fb, e, page_flip->flags);
         if (ret) {
                 if (page_flip->flags & DRM_MODE_PAGE_FLIP_EVENT) {
@@ -3719,7 +4222,7 @@ int drm_mode_page_flip_ioctl(struct drm_device *dev,
                  * Failing to do so will screw with the reference counting
                  * on framebuffers.
                  */
-                WARN_ON(crtc->fb != fb);
+                WARN_ON(crtc->primary->fb != fb);
                 /* Unref only the old framebuffer. */
                 fb = NULL;
         }
@@ -3734,6 +4237,14 @@ out:
         return ret;
 }
 
+/**
+ * drm_mode_config_reset - call ->reset callbacks
+ * @dev: drm device
+ *
+ * This functions calls all the crtc's, encoder's and connector's ->reset
+ * callback. Drivers can use this in e.g. their driver load or resume code to
+ * reset hardware and software state.
+ */
 void drm_mode_config_reset(struct drm_device *dev)
 {
         struct drm_crtc *crtc;
@@ -3757,16 +4268,66 @@ void drm_mode_config_reset(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_mode_config_reset);
 
+/**
+ * drm_mode_create_dumb_ioctl - create a dumb backing storage buffer
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * This creates a new dumb buffer in the driver's backing storage manager (GEM,
+ * TTM or something else entirely) and returns the resulting buffer handle. This
+ * handle can then be wrapped up into a framebuffer modeset object.
+ *
+ * Note that userspace is not allowed to use such objects for render
+ * acceleration - drivers must create their own private ioctls for such a use
+ * case.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_create_dumb_ioctl(struct drm_device *dev,
                                void *data, struct drm_file *file_priv)
 {
         struct drm_mode_create_dumb *args = data;
+        u32 cpp, stride, size;
 
         if (!dev->driver->dumb_create)
                 return -ENOSYS;
+        if (!args->width || !args->height || !args->bpp)
+                return -EINVAL;
+
+        /* overflow checks for 32bit size calculations */
+        cpp = DIV_ROUND_UP(args->bpp, 8);
+        if (cpp > 0xffffffffU / args->width)
+                return -EINVAL;
+        stride = cpp * args->width;
+        if (args->height > 0xffffffffU / stride)
+                return -EINVAL;
+
+        /* test for wrap-around */
+        size = args->height * stride;
+        if (PAGE_ALIGN(size) == 0)
+                return -EINVAL;
+
         return dev->driver->dumb_create(file_priv, dev, args);
 }
 
+/**
+ * drm_mode_mmap_dumb_ioctl - create an mmap offset for a dumb backing storage buffer
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * Allocate an offset in the drm device node's address space to be able to
+ * memory map a dumb buffer.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_mmap_dumb_ioctl(struct drm_device *dev,
                              void *data, struct drm_file *file_priv)
 {
@@ -3779,6 +4340,21 @@ int drm_mode_mmap_dumb_ioctl(struct drm_device *dev,
         return dev->driver->dumb_map_offset(file_priv, dev, args->handle, &args->offset);
 }
 
+/**
+ * drm_mode_destroy_dumb_ioctl - destroy a dumb backing strage buffer
+ * @dev: DRM device
+ * @data: ioctl data
+ * @file_priv: DRM file info
+ *
+ * This destroys the userspace handle for the given dumb backing storage buffer.
+ * Since buffer objects must be reference counted in the kernel a buffer object
+ * won't be immediately freed if a framebuffer modeset object still uses it.
+ *
+ * Called by the user via ioctl.
+ *
+ * Returns:
+ * Zero on success, errno on failure.
+ */
 int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
                                 void *data, struct drm_file *file_priv)
 {
@@ -3790,9 +4366,14 @@ int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
         return dev->driver->dumb_destroy(file_priv, dev, args->handle);
 }
 
-/*
- * Just need to support RGB formats here for compat with code that doesn't
- * use pixel formats directly yet.
+/**
+ * drm_fb_get_bpp_depth - get the bpp/depth values for format
+ * @format: pixel format (DRM_FORMAT_*)
+ * @depth: storage for the depth value
+ * @bpp: storage for the bpp value
+ *
+ * This only supports RGB formats here for compat with code that doesn't use
+ * pixel formats directly yet.
  */
 void drm_fb_get_bpp_depth(uint32_t format, unsigned int *depth,
                           int *bpp)
@@ -3864,7 +4445,7 @@ EXPORT_SYMBOL(drm_fb_get_bpp_depth);
  * drm_format_num_planes - get the number of planes for format
  * @format: pixel format (DRM_FORMAT_*)
  *
- * RETURNS:
+ * Returns:
  * The number of planes used by the specified pixel format.
  */
 int drm_format_num_planes(uint32_t format)
@@ -3899,7 +4480,7 @@ EXPORT_SYMBOL(drm_format_num_planes);
  * @format: pixel format (DRM_FORMAT_*)
  * @plane: plane index
  *
- * RETURNS:
+ * Returns:
  * The bytes per pixel value for the specified plane.
  */
 int drm_format_plane_cpp(uint32_t format, int plane)
@@ -3945,7 +4526,7 @@ EXPORT_SYMBOL(drm_format_plane_cpp);
  * drm_format_horz_chroma_subsampling - get the horizontal chroma subsampling factor
  * @format: pixel format (DRM_FORMAT_*)
  *
- * RETURNS:
+ * Returns:
  * The horizontal chroma subsampling factor for the
  * specified pixel format.
  */
@@ -3980,7 +4561,7 @@ EXPORT_SYMBOL(drm_format_horz_chroma_subsampling);
  * drm_format_vert_chroma_subsampling - get the vertical chroma subsampling factor
  * @format: pixel format (DRM_FORMAT_*)
  *
- * RETURNS:
+ * Returns:
  * The vertical chroma subsampling factor for the
  * specified pixel format.
  */
@@ -4030,6 +4611,7 @@ void drm_mode_config_init(struct drm_device *dev)
 
         drm_modeset_lock_all(dev);
         drm_mode_create_standard_connector_properties(dev);
+        drm_mode_create_standard_plane_properties(dev);
         drm_modeset_unlock_all(dev);
 
         /* Just to be sure */
@@ -4037,6 +4619,8 @@ void drm_mode_config_init(struct drm_device *dev)
         dev->mode_config.num_connector = 0;
         dev->mode_config.num_crtc = 0;
         dev->mode_config.num_encoder = 0;
+        dev->mode_config.num_overlay_plane = 0;
+        dev->mode_config.num_total_plane = 0;
 }
 EXPORT_SYMBOL(drm_mode_config_init);
 
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index f7a81209beb3..c43825e8f5c1 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -105,9 +105,6 @@ static void drm_mode_validate_flag(struct drm_connector *connector,
  * @maxX: max width for modes
  * @maxY: max height for modes
  *
- * LOCKING:
- * Caller must hold mode config lock.
- *
  * Based on the helper callbacks implemented by @connector try to detect all
  * valid modes.  Modes will first be added to the connector's probed_modes list,
  * then culled (based on validity and the @maxX, @maxY parameters) and put into
@@ -117,8 +114,8 @@ static void drm_mode_validate_flag(struct drm_connector *connector,
  * @connector vfunc for drivers that use the crtc helpers for output mode
  * filtering and detection.
  *
- * RETURNS:
- * Number of modes found on @connector.
+ * Returns:
+ * The number of modes found on @connector.
  */
 int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
                                             uint32_t maxX, uint32_t maxY)
@@ -131,6 +128,8 @@ int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
         int mode_flags = 0;
         bool verbose_prune = true;
 
+        WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
+
         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n", connector->base.id,
                         drm_get_connector_name(connector));
         /* set all modes to the unverified state */
@@ -176,8 +175,7 @@ int drm_helper_probe_single_connector_modes(struct drm_connector *connector,
         drm_mode_connector_list_update(connector);
 
         if (maxX && maxY)
-                drm_mode_validate_size(dev, &connector->modes, maxX,
-                                       maxY, 0);
+                drm_mode_validate_size(dev, &connector->modes, maxX, maxY);
 
         if (connector->interlace_allowed)
                 mode_flags |= DRM_MODE_FLAG_INTERLACE;
@@ -219,18 +217,19 @@ EXPORT_SYMBOL(drm_helper_probe_single_connector_modes);
  * drm_helper_encoder_in_use - check if a given encoder is in use
  * @encoder: encoder to check
  *
- * LOCKING:
- * Caller must hold mode config lock.
- *
- * Walk @encoders's DRM device's mode_config and see if it's in use.
+ * Checks whether @encoder is with the current mode setting output configuration
+ * in use by any connector. This doesn't mean that it is actually enabled since
+ * the DPMS state is tracked separately.
  *
- * RETURNS:
- * True if @encoder is part of the mode_config, false otherwise.
+ * Returns:
+ * True if @encoder is used, false otherwise.
  */
 bool drm_helper_encoder_in_use(struct drm_encoder *encoder)
 {
         struct drm_connector *connector;
         struct drm_device *dev = encoder->dev;
+
+        WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
         list_for_each_entry(connector, &dev->mode_config.connector_list, head)
                 if (connector->encoder == encoder)
                         return true;
@@ -242,19 +241,19 @@ EXPORT_SYMBOL(drm_helper_encoder_in_use);
  * drm_helper_crtc_in_use - check if a given CRTC is in a mode_config
  * @crtc: CRTC to check
  *
- * LOCKING:
- * Caller must hold mode config lock.
- *
- * Walk @crtc's DRM device's mode_config and see if it's in use.
+ * Checks whether @crtc is with the current mode setting output configuration
+ * in use by any connector. This doesn't mean that it is actually enabled since
+ * the DPMS state is tracked separately.
  *
- * RETURNS:
- * True if @crtc is part of the mode_config, false otherwise.
+ * Returns:
+ * True if @crtc is used, false otherwise.
  */
 bool drm_helper_crtc_in_use(struct drm_crtc *crtc)
 {
         struct drm_encoder *encoder;
         struct drm_device *dev = crtc->dev;
-        /* FIXME: Locking around list access? */
+
+        WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head)
                 if (encoder->crtc == crtc && drm_helper_encoder_in_use(encoder))
                         return true;
@@ -279,27 +278,17 @@ drm_encoder_disable(struct drm_encoder *encoder)
                 encoder->bridge->funcs->post_disable(encoder->bridge);
 }
 
-/**
- * drm_helper_disable_unused_functions - disable unused objects
- * @dev: DRM device
- *
- * LOCKING:
- * Caller must hold mode config lock.
- *
- * If an connector or CRTC isn't part of @dev's mode_config, it can be disabled
- * by calling its dpms function, which should power it off.
- */
-void drm_helper_disable_unused_functions(struct drm_device *dev)
+static void __drm_helper_disable_unused_functions(struct drm_device *dev)
 {
         struct drm_encoder *encoder;
         struct drm_connector *connector;
         struct drm_crtc *crtc;
 
+        drm_warn_on_modeset_not_all_locked(dev);
+
         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
                 if (!connector->encoder)
                         continue;
-                if (connector->status == connector_status_disconnected)
-                        connector->encoder = NULL;
         }
 
         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
@@ -318,10 +307,27 @@ void drm_helper_disable_unused_functions(struct drm_device *dev)
                                 (*crtc_funcs->disable)(crtc);
                         else
                                 (*crtc_funcs->dpms)(crtc, DRM_MODE_DPMS_OFF);
-                        crtc->fb = NULL;
+                        crtc->primary->fb = NULL;
                 }
         }
 }
+
+/**
+ * drm_helper_disable_unused_functions - disable unused objects
+ * @dev: DRM device
+ *
+ * This function walks through the entire mode setting configuration of @dev. It
+ * will remove any crtc links of unused encoders and encoder links of
+ * disconnected connectors. Then it will disable all unused encoders and crtcs
+ * either by calling their disable callback if available or by calling their
+ * dpms callback with DRM_MODE_DPMS_OFF.
+ */
+void drm_helper_disable_unused_functions(struct drm_device *dev)
+{
+        drm_modeset_lock_all(dev);
+        __drm_helper_disable_unused_functions(dev);
+        drm_modeset_unlock_all(dev);
+}
 EXPORT_SYMBOL(drm_helper_disable_unused_functions);
 
 /*
@@ -355,9 +361,6 @@ drm_crtc_prepare_encoders(struct drm_device *dev)
  * @y: vertical offset into the surface
  * @old_fb: old framebuffer, for cleanup
  *
- * LOCKING:
- * Caller must hold mode config lock.
- *
  * Try to set @mode on @crtc.  Give @crtc and its associated connectors a chance
  * to fixup or reject the mode prior to trying to set it. This is an internal
  * helper that drivers could e.g. use to update properties that require the
@@ -367,8 +370,8 @@ drm_crtc_prepare_encoders(struct drm_device *dev)
  * drm_crtc_helper_set_config() helper function to drive the mode setting
  * sequence.
  *
- * RETURNS:
- * True if the mode was set successfully, or false otherwise.
+ * Returns:
+ * True if the mode was set successfully, false otherwise.
  */
 bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
                               struct drm_display_mode *mode,
@@ -384,6 +387,8 @@ bool drm_crtc_helper_set_mode(struct drm_crtc *crtc,
         struct drm_encoder *encoder;
         bool ret = true;
 
+        drm_warn_on_modeset_not_all_locked(dev);
+
         saved_enabled = crtc->enabled;
         crtc->enabled = drm_helper_crtc_in_use(crtc);
         if (!crtc->enabled)
@@ -552,7 +557,7 @@ drm_crtc_helper_disable(struct drm_crtc *crtc)
                 }
         }
 
-        drm_helper_disable_unused_functions(dev);
+        __drm_helper_disable_unused_functions(dev);
         return 0;
 }
 
@@ -560,17 +565,14 @@ drm_crtc_helper_disable(struct drm_crtc *crtc)
  * drm_crtc_helper_set_config - set a new config from userspace
  * @set: mode set configuration
  *
- * LOCKING:
- * Caller must hold mode config lock.
- *
  * Setup a new configuration, provided by the upper layers (either an ioctl call
  * from userspace or internally e.g. from the fbdev support code) in @set, and
  * enable it. This is the main helper functions for drivers that implement
  * kernel mode setting with the crtc helper functions and the assorted
  * ->prepare(), ->modeset() and ->commit() helper callbacks.
  *
- * RETURNS:
- * Returns 0 on success, -ERRNO on failure.
+ * Returns:
+ * Returns 0 on success, negative errno numbers on failure.
  */
 int drm_crtc_helper_set_config(struct drm_mode_set *set)
 {
@@ -612,6 +614,8 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
 
         dev = set->crtc->dev;
 
+        drm_warn_on_modeset_not_all_locked(dev);
+
         /*
          * Allocate space for the backup of all (non-pointer) encoder and
          * connector data.
@@ -647,19 +651,19 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
         save_set.mode = &set->crtc->mode;
         save_set.x = set->crtc->x;
         save_set.y = set->crtc->y;
-        save_set.fb = set->crtc->fb;
+        save_set.fb = set->crtc->primary->fb;
 
         /* We should be able to check here if the fb has the same properties
          * and then just flip_or_move it */
-        if (set->crtc->fb != set->fb) {
+        if (set->crtc->primary->fb != set->fb) {
                 /* If we have no fb then treat it as a full mode set */
-                if (set->crtc->fb == NULL) {
+                if (set->crtc->primary->fb == NULL) {
                         DRM_DEBUG_KMS("crtc has no fb, full mode set\n");
                         mode_changed = true;
                 } else if (set->fb == NULL) {
                         mode_changed = true;
                 } else if (set->fb->pixel_format !=
-                           set->crtc->fb->pixel_format) {
+                           set->crtc->primary->fb->pixel_format) {
                         mode_changed = true;
                 } else
                         fb_changed = true;
@@ -689,12 +693,13 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                                 if (new_encoder == NULL)
                                         /* don't break so fail path works correct */
                                         fail = 1;
-                                break;
 
                                 if (connector->dpms != DRM_MODE_DPMS_ON) {
                                         DRM_DEBUG_KMS("connector dpms not on, full mode switch\n");
                                         mode_changed = true;
                                 }
+
+                                break;
                         }
                 }
 
@@ -760,13 +765,13 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                         DRM_DEBUG_KMS("attempting to set mode from"
                                         " userspace\n");
                         drm_mode_debug_printmodeline(set->mode);
-                        set->crtc->fb = set->fb;
+                        set->crtc->primary->fb = set->fb;
                         if (!drm_crtc_helper_set_mode(set->crtc, set->mode,
                                                       set->x, set->y,
                                                       save_set.fb)) {
                                 DRM_ERROR("failed to set mode on [CRTC:%d]\n",
                                           set->crtc->base.id);
-                                set->crtc->fb = save_set.fb;
+                                set->crtc->primary->fb = save_set.fb;
                                 ret = -EINVAL;
                                 goto fail;
                         }
@@ -777,17 +782,17 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
                                 set->connectors[i]->funcs->dpms(set->connectors[i], DRM_MODE_DPMS_ON);
                         }
                 }
-                drm_helper_disable_unused_functions(dev);
+                __drm_helper_disable_unused_functions(dev);
         } else if (fb_changed) {
                 set->crtc->x = set->x;
                 set->crtc->y = set->y;
-                set->crtc->fb = set->fb;
+                set->crtc->primary->fb = set->fb;
                 ret = crtc_funcs->mode_set_base(set->crtc,
                                                 set->x, set->y, save_set.fb);
                 if (ret != 0) {
                         set->crtc->x = save_set.x;
                         set->crtc->y = save_set.y;
-                        set->crtc->fb = save_set.fb;
+                        set->crtc->primary->fb = save_set.fb;
                         goto fail;
                 }
         }
@@ -924,8 +929,16 @@ void drm_helper_connector_dpms(struct drm_connector *connector, int mode)
 }
 EXPORT_SYMBOL(drm_helper_connector_dpms);
 
-int drm_helper_mode_fill_fb_struct(struct drm_framebuffer *fb,
-                                   struct drm_mode_fb_cmd2 *mode_cmd)
+/**
+ * drm_helper_mode_fill_fb_struct - fill out framebuffer metadata
+ * @fb: drm_framebuffer object to fill out
+ * @mode_cmd: metadata from the userspace fb creation request
+ *
+ * This helper can be used in a drivers fb_create callback to pre-fill the fb's
+ * metadata fields.
+ */
+void drm_helper_mode_fill_fb_struct(struct drm_framebuffer *fb,
+                                    struct drm_mode_fb_cmd2 *mode_cmd)
 {
         int i;
 
@@ -938,26 +951,47 @@ int drm_helper_mode_fill_fb_struct(struct drm_framebuffer *fb,
         drm_fb_get_bpp_depth(mode_cmd->pixel_format, &fb->depth,
                                     &fb->bits_per_pixel);
         fb->pixel_format = mode_cmd->pixel_format;
-
-        return 0;
 }
 EXPORT_SYMBOL(drm_helper_mode_fill_fb_struct);
 
-int drm_helper_resume_force_mode(struct drm_device *dev)
+/**
+ * drm_helper_resume_force_mode - force-restore mode setting configuration
+ * @dev: drm_device which should be restored
+ *
+ * Drivers which use the mode setting helpers can use this function to
+ * force-restore the mode setting configuration e.g. on resume or when something
+ * else might have trampled over the hw state (like some overzealous old BIOSen
+ * tended to do).
+ *
+ * This helper doesn't provide a error return value since restoring the old
+ * config should never fail due to resource allocation issues since the driver
+ * has successfully set the restored configuration already. Hence this should
+ * boil down to the equivalent of a few dpms on calls, which also don't provide
+ * an error code.
+ *
+ * Drivers where simply restoring an old configuration again might fail (e.g.
+ * due to slight differences in allocating shared resources when the
+ * configuration is restored in a different order than when userspace set it up)
+ * need to use their own restore logic.
+ */
+void drm_helper_resume_force_mode(struct drm_device *dev)
 {
         struct drm_crtc *crtc;
         struct drm_encoder *encoder;
         struct drm_crtc_helper_funcs *crtc_funcs;
-        int ret, encoder_dpms;
+        int encoder_dpms;
+        bool ret;
 
+        drm_modeset_lock_all(dev);
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 
                 if (!crtc->enabled)
                         continue;
 
                 ret = drm_crtc_helper_set_mode(crtc, &crtc->mode,
-                                               crtc->x, crtc->y, crtc->fb);
+                                               crtc->x, crtc->y, crtc->primary->fb);
 
+                /* Restoring the old config should never fail! */
                 if (ret == false)
                         DRM_ERROR("failed to set mode on crtc %p\n", crtc);
 
@@ -980,12 +1014,29 @@ int drm_helper_resume_force_mode(struct drm_device *dev)
                                                      drm_helper_choose_crtc_dpms(crtc));
                 }
         }
+
         /* disable the unused connectors while restoring the modesetting */
-        drm_helper_disable_unused_functions(dev);
-        return 0;
+        __drm_helper_disable_unused_functions(dev);
+        drm_modeset_unlock_all(dev);
 }
 EXPORT_SYMBOL(drm_helper_resume_force_mode);
 
+/**
+ * drm_kms_helper_hotplug_event - fire off KMS hotplug events
+ * @dev: drm_device whose connector state changed
+ *
+ * This function fires off the uevent for userspace and also calls the
+ * output_poll_changed function, which is most commonly used to inform the fbdev
+ * emulation code and allow it to update the fbcon output configuration.
+ *
+ * Drivers should call this from their hotplug handling code when a change is
+ * detected. Note that this function does not do any output detection of its
+ * own, like drm_helper_hpd_irq_event() does - this is assumed to be done by the
+ * driver already.
+ *
+ * This function must be called from process context with no mode
+ * setting locks held.
+ */
 void drm_kms_helper_hotplug_event(struct drm_device *dev)
 {
         /* send a uevent + call fbdev */
@@ -1054,6 +1105,16 @@ static void output_poll_execute(struct work_struct *work)
                 schedule_delayed_work(delayed_work, DRM_OUTPUT_POLL_PERIOD);
 }
 
+/**
+ * drm_kms_helper_poll_disable - disable output polling
+ * @dev: drm_device
+ *
+ * This function disables the output polling work.
+ *
+ * Drivers can call this helper from their device suspend implementation. It is
+ * not an error to call this even when output polling isn't enabled or arlready
+ * disabled.
+ */
 void drm_kms_helper_poll_disable(struct drm_device *dev)
 {
         if (!dev->mode_config.poll_enabled)
@@ -1062,6 +1123,16 @@ void drm_kms_helper_poll_disable(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_kms_helper_poll_disable);
 
+/**
+ * drm_kms_helper_poll_enable - re-enable output polling.
+ * @dev: drm_device
+ *
+ * This function re-enables the output polling work.
+ *
+ * Drivers can call this helper from their device resume implementation. It is
+ * an error to call this when the output polling support has not yet been set
+ * up.
+ */
 void drm_kms_helper_poll_enable(struct drm_device *dev)
 {
         bool poll = false;
@@ -1081,6 +1152,25 @@ void drm_kms_helper_poll_enable(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_kms_helper_poll_enable);
 
+/**
+ * drm_kms_helper_poll_init - initialize and enable output polling
+ * @dev: drm_device
+ *
+ * This function intializes and then also enables output polling support for
+ * @dev. Drivers which do not have reliable hotplug support in hardware can use
+ * this helper infrastructure to regularly poll such connectors for changes in
+ * their connection state.
+ *
+ * Drivers can control which connectors are polled by setting the
+ * DRM_CONNECTOR_POLL_CONNECT and DRM_CONNECTOR_POLL_DISCONNECT flags. On
+ * connectors where probing live outputs can result in visual distortion drivers
+ * should not set the DRM_CONNECTOR_POLL_DISCONNECT flag to avoid this.
+ * Connectors which have no flag or only DRM_CONNECTOR_POLL_HPD set are
+ * completely ignored by the polling logic.
+ *
+ * Note that a connector can be both polled and probed from the hotplug handler,
+ * in case the hotplug interrupt is known to be unreliable.
+ */
 void drm_kms_helper_poll_init(struct drm_device *dev)
 {
         INIT_DELAYED_WORK(&dev->mode_config.output_poll_work, output_poll_execute);
@@ -1090,12 +1180,39 @@ void drm_kms_helper_poll_init(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_kms_helper_poll_init);
 
+/**
+ * drm_kms_helper_poll_fini - disable output polling and clean it up
+ * @dev: drm_device
+ */
 void drm_kms_helper_poll_fini(struct drm_device *dev)
 {
         drm_kms_helper_poll_disable(dev);
 }
 EXPORT_SYMBOL(drm_kms_helper_poll_fini);
 
+/**
+ * drm_helper_hpd_irq_event - hotplug processing
+ * @dev: drm_device
+ *
+ * Drivers can use this helper function to run a detect cycle on all connectors
+ * which have the DRM_CONNECTOR_POLL_HPD flag set in their &polled member. All
+ * other connectors are ignored, which is useful to avoid reprobing fixed
+ * panels.
+ *
+ * This helper function is useful for drivers which can't or don't track hotplug
+ * interrupts for each connector.
+ *
+ * Drivers which support hotplug interrupts for each connector individually and
+ * which have a more fine-grained detect logic should bypass this code and
+ * directly call drm_kms_helper_hotplug_event() in case the connector state
+ * changed.
+ *
+ * This function must be called from process context with no mode
+ * setting locks held.
+ *
+ * Note that a connector can be both polled and probed from the hotplug handler,
+ * in case the hotplug interrupt is known to be unreliable.
+ */
 bool drm_helper_hpd_irq_event(struct drm_device *dev)
 {
         struct drm_connector *connector;
diff --git a/drivers/gpu/drm/drm_crtc_internal.h b/drivers/gpu/drm/drm_crtc_internal.h
new file mode 100644
index 000000000000..a2945ee6d675
--- /dev/null
+++ b/drivers/gpu/drm/drm_crtc_internal.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright © 2006 Keith Packard
+ * Copyright © 2007-2008 Dave Airlie
+ * Copyright © 2007-2008 Intel Corporation
+ *   Jesse Barnes <jesse.barnes@intel.com>
+ * Copyright © 2014 Intel Corporation
+ *   Daniel Vetter <daniel.vetter@ffwll.ch>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+/*
+ * This header file contains mode setting related functions and definitions
+ * which are only used within the drm module as internal implementation details
+ * and are not exported to drivers.
+ */
+
+int drm_mode_object_get(struct drm_device *dev,
+                        struct drm_mode_object *obj, uint32_t obj_type);
+void drm_mode_object_put(struct drm_device *dev,
+                         struct drm_mode_object *object);
+
diff --git a/drivers/gpu/drm/drm_dp_helper.c b/drivers/gpu/drm/drm_dp_helper.c
index 9e978aae8972..27671489477d 100644
--- a/drivers/gpu/drm/drm_dp_helper.c
+++ b/drivers/gpu/drm/drm_dp_helper.c
@@ -346,3 +346,399 @@ int drm_dp_bw_code_to_link_rate(u8 link_bw)
         }
 }
 EXPORT_SYMBOL(drm_dp_bw_code_to_link_rate);
+
+/**
+ * DOC: dp helpers
+ *
+ * The DisplayPort AUX channel is an abstraction to allow generic, driver-
+ * independent access to AUX functionality. Drivers can take advantage of
+ * this by filling in the fields of the drm_dp_aux structure.
+ *
+ * Transactions are described using a hardware-independent drm_dp_aux_msg
+ * structure, which is passed into a driver's .transfer() implementation.
+ * Both native and I2C-over-AUX transactions are supported.
+ */
+
+static int drm_dp_dpcd_access(struct drm_dp_aux *aux, u8 request,
+                              unsigned int offset, void *buffer, size_t size)
+{
+        struct drm_dp_aux_msg msg;
+        unsigned int retry;
+        int err;
+
+        memset(&msg, 0, sizeof(msg));
+        msg.address = offset;
+        msg.request = request;
+        msg.buffer = buffer;
+        msg.size = size;
+
+        /*
+         * The specification doesn't give any recommendation on how often to
+         * retry native transactions, so retry 7 times like for I2C-over-AUX
+         * transactions.
+         */
+        for (retry = 0; retry < 7; retry++) {
+                err = aux->transfer(aux, &msg);
+                if (err < 0) {
+                        if (err == -EBUSY)
+                                continue;
+
+                        return err;
+                }
+
+
+                switch (msg.reply & DP_AUX_NATIVE_REPLY_MASK) {
+                case DP_AUX_NATIVE_REPLY_ACK:
+                        if (err < size)
+                                return -EPROTO;
+                        return err;
+
+                case DP_AUX_NATIVE_REPLY_NACK:
+                        return -EIO;
+
+                case DP_AUX_NATIVE_REPLY_DEFER:
+                        usleep_range(400, 500);
+                        break;
+                }
+        }
+
+        DRM_DEBUG_KMS("too many retries, giving up\n");
+        return -EIO;
+}
+
+/**
+ * drm_dp_dpcd_read() - read a series of bytes from the DPCD
+ * @aux: DisplayPort AUX channel
+ * @offset: address of the (first) register to read
+ * @buffer: buffer to store the register values
+ * @size: number of bytes in @buffer
+ *
+ * Returns the number of bytes transferred on success, or a negative error
+ * code on failure. -EIO is returned if the request was NAKed by the sink or
+ * if the retry count was exceeded. If not all bytes were transferred, this
+ * function returns -EPROTO. Errors from the underlying AUX channel transfer
+ * function, with the exception of -EBUSY (which causes the transaction to
+ * be retried), are propagated to the caller.
+ */
+ssize_t drm_dp_dpcd_read(struct drm_dp_aux *aux, unsigned int offset,
+                         void *buffer, size_t size)
+{
+        return drm_dp_dpcd_access(aux, DP_AUX_NATIVE_READ, offset, buffer,
+                                  size);
+}
+EXPORT_SYMBOL(drm_dp_dpcd_read);
+
+/**
+ * drm_dp_dpcd_write() - write a series of bytes to the DPCD
+ * @aux: DisplayPort AUX channel
+ * @offset: address of the (first) register to write
+ * @buffer: buffer containing the values to write
+ * @size: number of bytes in @buffer
+ *
+ * Returns the number of bytes transferred on success, or a negative error
+ * code on failure. -EIO is returned if the request was NAKed by the sink or
+ * if the retry count was exceeded. If not all bytes were transferred, this
+ * function returns -EPROTO. Errors from the underlying AUX channel transfer
+ * function, with the exception of -EBUSY (which causes the transaction to
+ * be retried), are propagated to the caller.
+ */
+ssize_t drm_dp_dpcd_write(struct drm_dp_aux *aux, unsigned int offset,
+                          void *buffer, size_t size)
+{
+        return drm_dp_dpcd_access(aux, DP_AUX_NATIVE_WRITE, offset, buffer,
+                                  size);
+}
+EXPORT_SYMBOL(drm_dp_dpcd_write);
+
+/**
+ * drm_dp_dpcd_read_link_status() - read DPCD link status (bytes 0x202-0x207)
+ * @aux: DisplayPort AUX channel
+ * @status: buffer to store the link status in (must be at least 6 bytes)
+ *
+ * Returns the number of bytes transferred on success or a negative error
+ * code on failure.
+ */
+int drm_dp_dpcd_read_link_status(struct drm_dp_aux *aux,
+                                 u8 status[DP_LINK_STATUS_SIZE])
+{
+        return drm_dp_dpcd_read(aux, DP_LANE0_1_STATUS, status,
+                                DP_LINK_STATUS_SIZE);
+}
+EXPORT_SYMBOL(drm_dp_dpcd_read_link_status);
+
+/**
+ * drm_dp_link_probe() - probe a DisplayPort link for capabilities
+ * @aux: DisplayPort AUX channel
+ * @link: pointer to structure in which to return link capabilities
+ *
+ * The structure filled in by this function can usually be passed directly
+ * into drm_dp_link_power_up() and drm_dp_link_configure() to power up and
+ * configure the link based on the link's capabilities.
+ *
+ * Returns 0 on success or a negative error code on failure.
+ */
+int drm_dp_link_probe(struct drm_dp_aux *aux, struct drm_dp_link *link)
+{
+        u8 values[3];
+        int err;
+
+        memset(link, 0, sizeof(*link));
+
+        err = drm_dp_dpcd_read(aux, DP_DPCD_REV, values, sizeof(values));
+        if (err < 0)
+                return err;
+
+        link->revision = values[0];
+        link->rate = drm_dp_bw_code_to_link_rate(values[1]);
+        link->num_lanes = values[2] & DP_MAX_LANE_COUNT_MASK;
+
+        if (values[2] & DP_ENHANCED_FRAME_CAP)
+                link->capabilities |= DP_LINK_CAP_ENHANCED_FRAMING;
+
+        return 0;
+}
+EXPORT_SYMBOL(drm_dp_link_probe);
+
+/**
+ * drm_dp_link_power_up() - power up a DisplayPort link
+ * @aux: DisplayPort AUX channel
+ * @link: pointer to a structure containing the link configuration
+ *
+ * Returns 0 on success or a negative error code on failure.
+ */
+int drm_dp_link_power_up(struct drm_dp_aux *aux, struct drm_dp_link *link)
+{
+        u8 value;
+        int err;
+
+        /* DP_SET_POWER register is only available on DPCD v1.1 and later */
+        if (link->revision < 0x11)
+                return 0;
+
+        err = drm_dp_dpcd_readb(aux, DP_SET_POWER, &value);
+        if (err < 0)
+                return err;
+
+        value &= ~DP_SET_POWER_MASK;
+        value |= DP_SET_POWER_D0;
+
+        err = drm_dp_dpcd_writeb(aux, DP_SET_POWER, value);
+        if (err < 0)
+                return err;
+
+        /*
+         * According to the DP 1.1 specification, a "Sink Device must exit the
+         * power saving state within 1 ms" (Section 2.5.3.1, Table 5-52, "Sink
+         * Control Field" (register 0x600).
+         */
+        usleep_range(1000, 2000);
+
+        return 0;
+}
+EXPORT_SYMBOL(drm_dp_link_power_up);
+
+/**
+ * drm_dp_link_configure() - configure a DisplayPort link
+ * @aux: DisplayPort AUX channel
+ * @link: pointer to a structure containing the link configuration
+ *
+ * Returns 0 on success or a negative error code on failure.
+ */
+int drm_dp_link_configure(struct drm_dp_aux *aux, struct drm_dp_link *link)
+{
+        u8 values[2];
+        int err;
+
+        values[0] = drm_dp_link_rate_to_bw_code(link->rate);
+        values[1] = link->num_lanes;
+
+        if (link->capabilities & DP_LINK_CAP_ENHANCED_FRAMING)
+                values[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
+
+        err = drm_dp_dpcd_write(aux, DP_LINK_BW_SET, values, sizeof(values));
+        if (err < 0)
+                return err;
+
+        return 0;
+}
+EXPORT_SYMBOL(drm_dp_link_configure);
+
+/*
+ * I2C-over-AUX implementation
+ */
+
+static u32 drm_dp_i2c_functionality(struct i2c_adapter *adapter)
+{
+        return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL |
+               I2C_FUNC_SMBUS_READ_BLOCK_DATA |
+               I2C_FUNC_SMBUS_BLOCK_PROC_CALL |
+               I2C_FUNC_10BIT_ADDR;
+}
+
+/*
+ * Transfer a single I2C-over-AUX message and handle various error conditions,
+ * retrying the transaction as appropriate.
+ */
+static int drm_dp_i2c_do_msg(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
+{
+        unsigned int retry;
+        int err;
+
+        /*
+         * DP1.2 sections 2.7.7.1.5.6.1 and 2.7.7.1.6.6.1: A DP Source device
+         * is required to retry at least seven times upon receiving AUX_DEFER
+         * before giving up the AUX transaction.
+         */
+        for (retry = 0; retry < 7; retry++) {
+                err = aux->transfer(aux, msg);
+                if (err < 0) {
+                        if (err == -EBUSY)
+                                continue;
+
+                        DRM_DEBUG_KMS("transaction failed: %d\n", err);
+                        return err;
+                }
+
+
+                switch (msg->reply & DP_AUX_NATIVE_REPLY_MASK) {
+                case DP_AUX_NATIVE_REPLY_ACK:
+                        /*
+                         * For I2C-over-AUX transactions this isn't enough, we
+                         * need to check for the I2C ACK reply.
+                         */
+                        break;
+
+                case DP_AUX_NATIVE_REPLY_NACK:
+                        DRM_DEBUG_KMS("native nack\n");
+                        return -EREMOTEIO;
+
+                case DP_AUX_NATIVE_REPLY_DEFER:
+                        DRM_DEBUG_KMS("native defer");
+                        /*
+                         * We could check for I2C bit rate capabilities and if
+                         * available adjust this interval. We could also be
+                         * more careful with DP-to-legacy adapters where a
+                         * long legacy cable may force very low I2C bit rates.
+                         *
+                         * For now just defer for long enough to hopefully be
+                         * safe for all use-cases.
+                         */
+                        usleep_range(500, 600);
+                        continue;
+
+                default:
+                        DRM_ERROR("invalid native reply %#04x\n", msg->reply);
+                        return -EREMOTEIO;
+                }
+
+                switch (msg->reply & DP_AUX_I2C_REPLY_MASK) {
+                case DP_AUX_I2C_REPLY_ACK:
+                        /*
+                         * Both native ACK and I2C ACK replies received. We
+                         * can assume the transfer was successful.
+                         */
+                        if (err < msg->size)
+                                return -EPROTO;
+                        return 0;
+
+                case DP_AUX_I2C_REPLY_NACK:
+                        DRM_DEBUG_KMS("I2C nack\n");
+                        return -EREMOTEIO;
+
+                case DP_AUX_I2C_REPLY_DEFER:
+                        DRM_DEBUG_KMS("I2C defer\n");
+                        usleep_range(400, 500);
+                        continue;
+
+                default:
+                        DRM_ERROR("invalid I2C reply %#04x\n", msg->reply);
+                        return -EREMOTEIO;
+                }
+        }
+
+        DRM_DEBUG_KMS("too many retries, giving up\n");
+        return -EREMOTEIO;
+}
+
+static int drm_dp_i2c_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs,
+                           int num)
+{
+        struct drm_dp_aux *aux = adapter->algo_data;
+        unsigned int i, j;
+
+        for (i = 0; i < num; i++) {
+                struct drm_dp_aux_msg msg;
+                int err;
+
+                /*
+                 * Many hardware implementations support FIFOs larger than a
+                 * single byte, but it has been empirically determined that
+                 * transferring data in larger chunks can actually lead to
+                 * decreased performance. Therefore each message is simply
+                 * transferred byte-by-byte.
+                 */
+                for (j = 0; j < msgs[i].len; j++) {
+                        memset(&msg, 0, sizeof(msg));
+                        msg.address = msgs[i].addr;
+
+                        msg.request = (msgs[i].flags & I2C_M_RD) ?
+                                        DP_AUX_I2C_READ :
+                                        DP_AUX_I2C_WRITE;
+
+                        /*
+                         * All messages except the last one are middle-of-
+                         * transfer messages.
+                         */
+                        if ((i < num - 1) || (j < msgs[i].len - 1))
+                                msg.request |= DP_AUX_I2C_MOT;
+
+                        msg.buffer = msgs[i].buf + j;
+                        msg.size = 1;
+
+                        err = drm_dp_i2c_do_msg(aux, &msg);
+                        if (err < 0)
+                                return err;
+                }
+        }
+
+        return num;
+}
+
+static const struct i2c_algorithm drm_dp_i2c_algo = {
+        .functionality = drm_dp_i2c_functionality,
+        .master_xfer = drm_dp_i2c_xfer,
+};
+
+/**
+ * drm_dp_aux_register_i2c_bus() - register an I2C adapter for I2C-over-AUX
+ * @aux: DisplayPort AUX channel
+ *
+ * Returns 0 on success or a negative error code on failure.
+ */
+int drm_dp_aux_register_i2c_bus(struct drm_dp_aux *aux)
+{
+        aux->ddc.algo = &drm_dp_i2c_algo;
+        aux->ddc.algo_data = aux;
+        aux->ddc.retries = 3;
+
+        aux->ddc.class = I2C_CLASS_DDC;
+        aux->ddc.owner = THIS_MODULE;
+        aux->ddc.dev.parent = aux->dev;
+        aux->ddc.dev.of_node = aux->dev->of_node;
+
+        strlcpy(aux->ddc.name, aux->name ? aux->name : dev_name(aux->dev),
+                sizeof(aux->ddc.name));
+
+        return i2c_add_adapter(&aux->ddc);
+}
+EXPORT_SYMBOL(drm_dp_aux_register_i2c_bus);
+
+/**
+ * drm_dp_aux_unregister_i2c_bus() - unregister an I2C-over-AUX adapter
+ * @aux: DisplayPort AUX channel
+ */
+void drm_dp_aux_unregister_i2c_bus(struct drm_dp_aux *aux)
+{
+        i2c_del_adapter(&aux->ddc);
+}
+EXPORT_SYMBOL(drm_dp_aux_unregister_i2c_bus);
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index 345be03c23db..03711d00aaae 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -286,6 +286,45 @@ static int drm_version(struct drm_device *dev, void *data,
 }
 
 /**
+ * drm_ioctl_permit - Check ioctl permissions against caller
+ *
+ * @flags: ioctl permission flags.
+ * @file_priv: Pointer to struct drm_file identifying the caller.
+ *
+ * Checks whether the caller is allowed to run an ioctl with the
+ * indicated permissions. If so, returns zero. Otherwise returns an
+ * error code suitable for ioctl return.
+ */
+static int drm_ioctl_permit(u32 flags, struct drm_file *file_priv)
+{
+        /* ROOT_ONLY is only for CAP_SYS_ADMIN */
+        if (unlikely((flags & DRM_ROOT_ONLY) && !capable(CAP_SYS_ADMIN)))
+                return -EACCES;
+
+        /* AUTH is only for authenticated or render client */
+        if (unlikely((flags & DRM_AUTH) && !drm_is_render_client(file_priv) &&
+                     !file_priv->authenticated))
+                return -EACCES;
+
+        /* MASTER is only for master or control clients */
+        if (unlikely((flags & DRM_MASTER) && !file_priv->is_master &&
+                     !drm_is_control_client(file_priv)))
+                return -EACCES;
+
+        /* Control clients must be explicitly allowed */
+        if (unlikely(!(flags & DRM_CONTROL_ALLOW) &&
+                     drm_is_control_client(file_priv)))
+                return -EACCES;
+
+        /* Render clients must be explicitly allowed */
+        if (unlikely(!(flags & DRM_RENDER_ALLOW) &&
+                     drm_is_render_client(file_priv)))
+                return -EACCES;
+
+        return 0;
+}
+
+/**
  * Called whenever a process performs an ioctl on /dev/drm.
  *
  * \param inode device inode.
@@ -344,65 +383,64 @@ long drm_ioctl(struct file *filp,
 
         DRM_DEBUG("pid=%d, dev=0x%lx, auth=%d, %s\n",
                   task_pid_nr(current),
-                  (long)old_encode_dev(file_priv->minor->device),
+                  (long)old_encode_dev(file_priv->minor->kdev->devt),
                   file_priv->authenticated, ioctl->name);
 
         /* Do not trust userspace, use our own definition */
         func = ioctl->func;
 
-        if (!func) {
+        if (unlikely(!func)) {
                 DRM_DEBUG("no function\n");
                 retcode = -EINVAL;
-        } else if (((ioctl->flags & DRM_ROOT_ONLY) && !capable(CAP_SYS_ADMIN)) ||
-                   ((ioctl->flags & DRM_AUTH) && !drm_is_render_client(file_priv) && !file_priv->authenticated) ||
-                   ((ioctl->flags & DRM_MASTER) && !file_priv->is_master) ||
-                   (!(ioctl->flags & DRM_CONTROL_ALLOW) && (file_priv->minor->type == DRM_MINOR_CONTROL)) ||
-                   (!(ioctl->flags & DRM_RENDER_ALLOW) && drm_is_render_client(file_priv))) {
-                retcode = -EACCES;
-        } else {
-                if (cmd & (IOC_IN | IOC_OUT)) {
-                        if (asize <= sizeof(stack_kdata)) {
-                                kdata = stack_kdata;
-                        } else {
-                                kdata = kmalloc(asize, GFP_KERNEL);
-                                if (!kdata) {
-                                        retcode = -ENOMEM;
-                                        goto err_i1;
-                                }
-                        }
-                        if (asize > usize)
-                                memset(kdata + usize, 0, asize - usize);
-                }
+                goto err_i1;
+        }
 
-                if (cmd & IOC_IN) {
-                        if (copy_from_user(kdata, (void __user *)arg,
-                                           usize) != 0) {
-                                retcode = -EFAULT;
+        retcode = drm_ioctl_permit(ioctl->flags, file_priv);
+        if (unlikely(retcode))
+                goto err_i1;
+
+        if (cmd & (IOC_IN | IOC_OUT)) {
+                if (asize <= sizeof(stack_kdata)) {
+                        kdata = stack_kdata;
+                } else {
+                        kdata = kmalloc(asize, GFP_KERNEL);
+                        if (!kdata) {
+                                retcode = -ENOMEM;
                                 goto err_i1;
                         }
-                } else
-                        memset(kdata, 0, usize);
-
-                if (ioctl->flags & DRM_UNLOCKED)
-                        retcode = func(dev, kdata, file_priv);
-                else {
-                        mutex_lock(&drm_global_mutex);
-                        retcode = func(dev, kdata, file_priv);
-                        mutex_unlock(&drm_global_mutex);
                 }
+                if (asize > usize)
+                        memset(kdata + usize, 0, asize - usize);
+        }
 
-                if (cmd & IOC_OUT) {
-                        if (copy_to_user((void __user *)arg, kdata,
-                                         usize) != 0)
-                                retcode = -EFAULT;
+        if (cmd & IOC_IN) {
+                if (copy_from_user(kdata, (void __user *)arg,
+                                   usize) != 0) {
+                        retcode = -EFAULT;
+                        goto err_i1;
                 }
+        } else
+                memset(kdata, 0, usize);
+
+        if (ioctl->flags & DRM_UNLOCKED)
+                retcode = func(dev, kdata, file_priv);
+        else {
+                mutex_lock(&drm_global_mutex);
+                retcode = func(dev, kdata, file_priv);
+                mutex_unlock(&drm_global_mutex);
+        }
+
+        if (cmd & IOC_OUT) {
+                if (copy_to_user((void __user *)arg, kdata,
+                                 usize) != 0)
+                        retcode = -EFAULT;
         }
 
       err_i1:
         if (!ioctl)
                 DRM_DEBUG("invalid ioctl: pid=%d, dev=0x%lx, auth=%d, cmd=0x%02x, nr=0x%02x\n",
                           task_pid_nr(current),
-                          (long)old_encode_dev(file_priv->minor->device),
+                          (long)old_encode_dev(file_priv->minor->kdev->devt),
                           file_priv->authenticated, cmd, nr);
 
         if (kdata != stack_kdata)
@@ -412,3 +450,21 @@ long drm_ioctl(struct file *filp,
         return retcode;
 }
 EXPORT_SYMBOL(drm_ioctl);
+
+/**
+ * drm_ioctl_flags - Check for core ioctl and return ioctl permission flags
+ *
+ * @nr: Ioctl number.
+ * @flags: Where to return the ioctl permission flags
+ */
+bool drm_ioctl_flags(unsigned int nr, unsigned int *flags)
+{
+        if ((nr >= DRM_COMMAND_END && nr < DRM_CORE_IOCTL_COUNT) ||
+            (nr < DRM_COMMAND_BASE)) {
+                *flags = drm_ioctls[nr].flags;
+                return true;
+        }
+
+        return false;
+}
+EXPORT_SYMBOL(drm_ioctl_flags);
diff --git a/drivers/gpu/drm/drm_edid.c b/drivers/gpu/drm/drm_edid.c
index b924306b8477..d4e3f9d9370f 100644
--- a/drivers/gpu/drm/drm_edid.c
+++ b/drivers/gpu/drm/drm_edid.c
@@ -1098,10 +1098,14 @@ EXPORT_SYMBOL(drm_edid_is_valid);
 /**
  * Get EDID information via I2C.
  *
- * \param adapter : i2c device adaptor
- * \param buf     : EDID data buffer to be filled
- * \param len     : EDID data buffer length
- * \return 0 on success or -1 on failure.
+ * @adapter : i2c device adaptor
+ * @buf: EDID data buffer to be filled
+ * @block: 128 byte EDID block to start fetching from
+ * @len: EDID data buffer length to fetch
+ *
+ * Returns:
+ *
+ * 0 on success or -1 on failure.
  *
  * Try to fetch EDID information by calling i2c driver function.
  */
@@ -1243,9 +1247,11 @@ out:
 
 /**
  * Probe DDC presence.
+ * @adapter: i2c adapter to probe
+ *
+ * Returns:
  *
- * \param adapter : i2c device adaptor
- * \return 1 on success
+ * 1 on success
  */
 bool
 drm_probe_ddc(struct i2c_adapter *adapter)
@@ -1586,8 +1592,10 @@ bad_std_timing(u8 a, u8 b)
 
 /**
  * drm_mode_std - convert standard mode info (width, height, refresh) into mode
+ * @connector: connector of for the EDID block
+ * @edid: EDID block to scan
  * @t: standard timing params
- * @timing_level: standard timing level
+ * @revision: standard timing level
  *
  * Take the standard timing params (in this case width, aspect, and refresh)
  * and convert them into a real mode using CVT/GTF/DMT.
@@ -2132,6 +2140,7 @@ do_established_modes(struct detailed_timing *timing, void *c)
 
 /**
  * add_established_modes - get est. modes from EDID and add them
+ * @connector: connector of for the EDID block
  * @edid: EDID block to scan
  *
  * Each EDID block contains a bitmap of the supported "established modes" list
@@ -2194,6 +2203,7 @@ do_standard_modes(struct detailed_timing *timing, void *c)
 
 /**
  * add_standard_modes - get std. modes from EDID and add them
+ * @connector: connector of for the EDID block
  * @edid: EDID block to scan
  *
  * Standard modes can be calculated using the appropriate standard (DMT,
@@ -2580,6 +2590,9 @@ drm_display_mode_from_vic_index(struct drm_connector *connector,
                 return NULL;
 
         newmode = drm_mode_duplicate(dev, &edid_cea_modes[cea_mode]);
+        if (!newmode)
+                return NULL;
+
         newmode->vrefresh = 0;
 
         return newmode;
@@ -3300,6 +3313,7 @@ EXPORT_SYMBOL(drm_detect_hdmi_monitor);
 
 /**
  * drm_detect_monitor_audio - check monitor audio capability
+ * @edid: EDID block to scan
  *
  * Monitor should have CEA extension block.
  * If monitor has 'basic audio', but no CEA audio blocks, it's 'basic
@@ -3345,6 +3359,7 @@ EXPORT_SYMBOL(drm_detect_monitor_audio);
 
 /**
  * drm_rgb_quant_range_selectable - is RGB quantization range selectable?
+ * @edid: EDID block to scan
  *
  * Check whether the monitor reports the RGB quantization range selection
  * as supported. The AVI infoframe can then be used to inform the monitor
@@ -3564,8 +3579,8 @@ void drm_set_preferred_mode(struct drm_connector *connector,
         struct drm_display_mode *mode;
 
         list_for_each_entry(mode, &connector->probed_modes, head) {
-                if (drm_mode_width(mode)  == hpref &&
-                    drm_mode_height(mode) == vpref)
+                if (mode->hdisplay  == hpref &&
+                    mode->vdisplay == vpref)
                         mode->type |= DRM_MODE_TYPE_PREFERRED;
         }
 }
@@ -3599,6 +3614,7 @@ drm_hdmi_avi_infoframe_from_display_mode(struct hdmi_avi_infoframe *frame,
 
         frame->picture_aspect = HDMI_PICTURE_ASPECT_NONE;
         frame->active_aspect = HDMI_ACTIVE_ASPECT_PICTURE;
+        frame->scan_mode = HDMI_SCAN_MODE_UNDERSCAN;
 
         return 0;
 }
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index 98a03639b413..04d3fd3658f3 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -232,7 +232,7 @@ static struct drm_framebuffer *drm_mode_config_fb(struct drm_crtc *crtc)
 
         list_for_each_entry(c, &dev->mode_config.crtc_list, head) {
                 if (crtc->base.id == c->base.id)
-                        return c->fb;
+                        return c->primary->fb;
         }
 
         return NULL;
@@ -291,7 +291,8 @@ bool drm_fb_helper_restore_fbdev_mode(struct drm_fb_helper *fb_helper)
         drm_warn_on_modeset_not_all_locked(dev);
 
         list_for_each_entry(plane, &dev->mode_config.plane_list, head)
-                drm_plane_force_disable(plane);
+                if (plane->type != DRM_PLANE_TYPE_PRIMARY)
+                        drm_plane_force_disable(plane);
 
         for (i = 0; i < fb_helper->crtc_count; i++) {
                 struct drm_mode_set *mode_set = &fb_helper->crtc_info[i].mode_set;
@@ -365,9 +366,9 @@ static bool drm_fb_helper_is_bound(struct drm_fb_helper *fb_helper)
                 return false;
 
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-                if (crtc->fb)
+                if (crtc->primary->fb)
                         crtcs_bound++;
-                if (crtc->fb == fb_helper->fb)
+                if (crtc->primary->fb == fb_helper->fb)
                         bound++;
         }
 
@@ -516,6 +517,9 @@ int drm_fb_helper_init(struct drm_device *dev,
         struct drm_crtc *crtc;
         int i;
 
+        if (!max_conn_count)
+                return -EINVAL;
+
         fb_helper->dev = dev;
 
         INIT_LIST_HEAD(&fb_helper->kernel_fb_list);
@@ -809,8 +813,6 @@ int drm_fb_helper_set_par(struct fb_info *info)
         struct drm_fb_helper *fb_helper = info->par;
         struct drm_device *dev = fb_helper->dev;
         struct fb_var_screeninfo *var = &info->var;
-        int ret;
-        int i;
 
         if (var->pixclock != 0) {
                 DRM_ERROR("PIXEL CLOCK SET\n");
@@ -818,13 +820,7 @@ int drm_fb_helper_set_par(struct fb_info *info)
         }
 
         drm_modeset_lock_all(dev);
-        for (i = 0; i < fb_helper->crtc_count; i++) {
-                ret = drm_mode_set_config_internal(&fb_helper->crtc_info[i].mode_set);
-                if (ret) {
-                        drm_modeset_unlock_all(dev);
-                        return ret;
-                }
-        }
+        drm_fb_helper_restore_fbdev_mode(fb_helper);
         drm_modeset_unlock_all(dev);
 
         if (fb_helper->delayed_hotplug) {
@@ -1136,19 +1132,20 @@ static int drm_fb_helper_probe_connector_modes(struct drm_fb_helper *fb_helper,
         return count;
 }
 
-static struct drm_display_mode *drm_has_preferred_mode(struct drm_fb_helper_connector *fb_connector, int width, int height)
+struct drm_display_mode *drm_has_preferred_mode(struct drm_fb_helper_connector *fb_connector, int width, int height)
 {
         struct drm_display_mode *mode;
 
         list_for_each_entry(mode, &fb_connector->connector->modes, head) {
-                if (drm_mode_width(mode) > width ||
-                    drm_mode_height(mode) > height)
+                if (mode->hdisplay > width ||
+                    mode->vdisplay > height)
                         continue;
                 if (mode->type & DRM_MODE_TYPE_PREFERRED)
                         return mode;
         }
         return NULL;
 }
+EXPORT_SYMBOL(drm_has_preferred_mode);
 
 static bool drm_has_cmdline_mode(struct drm_fb_helper_connector *fb_connector)
 {
@@ -1157,11 +1154,12 @@ static bool drm_has_cmdline_mode(struct drm_fb_helper_connector *fb_connector)
         return cmdline_mode->specified;
 }
 
-static struct drm_display_mode *drm_pick_cmdline_mode(struct drm_fb_helper_connector *fb_helper_conn,
+struct drm_display_mode *drm_pick_cmdline_mode(struct drm_fb_helper_connector *fb_helper_conn,
                                                       int width, int height)
 {
         struct drm_cmdline_mode *cmdline_mode;
         struct drm_display_mode *mode = NULL;
+        bool prefer_non_interlace;
 
         cmdline_mode = &fb_helper_conn->cmdline_mode;
         if (cmdline_mode->specified == false)
@@ -1173,6 +1171,8 @@ static struct drm_display_mode *drm_pick_cmdline_mode(struct drm_fb_helper_conne
         if (cmdline_mode->rb || cmdline_mode->margins)
                 goto create_mode;
 
+        prefer_non_interlace = !cmdline_mode->interlace;
+ again:
         list_for_each_entry(mode, &fb_helper_conn->connector->modes, head) {
                 /* check width/height */
                 if (mode->hdisplay != cmdline_mode->xres ||
@@ -1187,16 +1187,25 @@ static struct drm_display_mode *drm_pick_cmdline_mode(struct drm_fb_helper_conne
                 if (cmdline_mode->interlace) {
                         if (!(mode->flags & DRM_MODE_FLAG_INTERLACE))
                                 continue;
+                } else if (prefer_non_interlace) {
+                        if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+                                continue;
                 }
                 return mode;
         }
 
+        if (prefer_non_interlace) {
+                prefer_non_interlace = false;
+                goto again;
+        }
+
 create_mode:
         mode = drm_mode_create_from_cmdline_mode(fb_helper_conn->connector->dev,
                                                  cmdline_mode);
         list_add(&mode->head, &fb_helper_conn->connector->modes);
         return mode;
 }
+EXPORT_SYMBOL(drm_pick_cmdline_mode);
 
 static bool drm_connector_enabled(struct drm_connector *connector, bool strict)
 {
@@ -1539,9 +1548,11 @@ bool drm_fb_helper_initial_config(struct drm_fb_helper *fb_helper, int bpp_sel)
 
         drm_fb_helper_parse_command_line(fb_helper);
 
+        mutex_lock(&dev->mode_config.mutex);
         count = drm_fb_helper_probe_connector_modes(fb_helper,
                                                     dev->mode_config.max_width,
                                                     dev->mode_config.max_height);
+        mutex_unlock(&dev->mode_config.mutex);
         /*
          * we shouldn't end up with no modes here.
          */
diff --git a/drivers/gpu/drm/drm_fops.c b/drivers/gpu/drm/drm_fops.c
index 309023f12d7f..e1eba0b7cd45 100644
--- a/drivers/gpu/drm/drm_fops.c
+++ b/drivers/gpu/drm/drm_fops.c
@@ -39,12 +39,12 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 
-/* from BKL pushdown: note that nothing else serializes idr_find() */
+/* from BKL pushdown */
 DEFINE_MUTEX(drm_global_mutex);
 EXPORT_SYMBOL(drm_global_mutex);
 
 static int drm_open_helper(struct inode *inode, struct file *filp,
-                           struct drm_device * dev);
+                           struct drm_minor *minor);
 
 static int drm_setup(struct drm_device * dev)
 {
@@ -79,38 +79,23 @@ static int drm_setup(struct drm_device * dev)
  */
 int drm_open(struct inode *inode, struct file *filp)
 {
-        struct drm_device *dev = NULL;
-        int minor_id = iminor(inode);
+        struct drm_device *dev;
         struct drm_minor *minor;
-        int retcode = 0;
+        int retcode;
         int need_setup = 0;
-        struct address_space *old_mapping;
-        struct address_space *old_imapping;
-
-        minor = idr_find(&drm_minors_idr, minor_id);
-        if (!minor)
-                return -ENODEV;
 
-        if (!(dev = minor->dev))
-                return -ENODEV;
-
-        if (drm_device_is_unplugged(dev))
-                return -ENODEV;
+        minor = drm_minor_acquire(iminor(inode));
+        if (IS_ERR(minor))
+                return PTR_ERR(minor);
 
+        dev = minor->dev;
         if (!dev->open_count++)
                 need_setup = 1;
-        mutex_lock(&dev->struct_mutex);
-        old_imapping = inode->i_mapping;
-        old_mapping = dev->dev_mapping;
-        if (old_mapping == NULL)
-                dev->dev_mapping = &inode->i_data;
-        /* ihold ensures nobody can remove inode with our i_data */
-        ihold(container_of(dev->dev_mapping, struct inode, i_data));
-        inode->i_mapping = dev->dev_mapping;
-        filp->f_mapping = dev->dev_mapping;
-        mutex_unlock(&dev->struct_mutex);
 
-        retcode = drm_open_helper(inode, filp, dev);
+        /* share address_space across all char-devs of a single device */
+        filp->f_mapping = dev->anon_inode->i_mapping;
+
+        retcode = drm_open_helper(inode, filp, minor);
         if (retcode)
                 goto err_undo;
         if (need_setup) {
@@ -121,13 +106,8 @@ int drm_open(struct inode *inode, struct file *filp)
         return 0;
 
 err_undo:
-        mutex_lock(&dev->struct_mutex);
-        filp->f_mapping = old_imapping;
-        inode->i_mapping = old_imapping;
-        iput(container_of(dev->dev_mapping, struct inode, i_data));
-        dev->dev_mapping = old_mapping;
-        mutex_unlock(&dev->struct_mutex);
         dev->open_count--;
+        drm_minor_release(minor);
         return retcode;
 }
 EXPORT_SYMBOL(drm_open);
@@ -143,33 +123,30 @@ EXPORT_SYMBOL(drm_open);
  */
 int drm_stub_open(struct inode *inode, struct file *filp)
 {
-        struct drm_device *dev = NULL;
+        struct drm_device *dev;
         struct drm_minor *minor;
-        int minor_id = iminor(inode);
         int err = -ENODEV;
         const struct file_operations *new_fops;
 
         DRM_DEBUG("\n");
 
         mutex_lock(&drm_global_mutex);
-        minor = idr_find(&drm_minors_idr, minor_id);
-        if (!minor)
-                goto out;
-
-        if (!(dev = minor->dev))
-                goto out;
-
-        if (drm_device_is_unplugged(dev))
-                goto out;
+        minor = drm_minor_acquire(iminor(inode));
+        if (IS_ERR(minor))
+                goto out_unlock;
 
+        dev = minor->dev;
         new_fops = fops_get(dev->driver->fops);
         if (!new_fops)
-                goto out;
+                goto out_release;
 
         replace_fops(filp, new_fops);
         if (filp->f_op->open)
                 err = filp->f_op->open(inode, filp);
-out:
+
+out_release:
+        drm_minor_release(minor);
+out_unlock:
         mutex_unlock(&drm_global_mutex);
         return err;
 }
@@ -196,16 +173,16 @@ static int drm_cpu_valid(void)
  *
  * \param inode device inode.
  * \param filp file pointer.
- * \param dev device.
+ * \param minor acquired minor-object.
  * \return zero on success or a negative number on failure.
  *
  * Creates and initializes a drm_file structure for the file private data in \p
  * filp and add it into the double linked list in \p dev.
  */
 static int drm_open_helper(struct inode *inode, struct file *filp,
-                           struct drm_device * dev)
+                           struct drm_minor *minor)
 {
-        int minor_id = iminor(inode);
+        struct drm_device *dev = minor->dev;
         struct drm_file *priv;
         int ret;
 
@@ -216,7 +193,7 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
         if (dev->switch_power_state != DRM_SWITCH_POWER_ON && dev->switch_power_state != DRM_SWITCH_POWER_DYNAMIC_OFF)
                 return -EINVAL;
 
-        DRM_DEBUG("pid = %d, minor = %d\n", task_pid_nr(current), minor_id);
+        DRM_DEBUG("pid = %d, minor = %d\n", task_pid_nr(current), minor->index);
 
         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
         if (!priv)
@@ -226,11 +203,7 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
         priv->filp = filp;
         priv->uid = current_euid();
         priv->pid = get_pid(task_pid(current));
-        priv->minor = idr_find(&drm_minors_idr, minor_id);
-        if (!priv->minor) {
-                ret = -ENODEV;
-                goto out_put_pid;
-        }
+        priv->minor = minor;
 
         /* for compatibility root is always authenticated */
         priv->always_authenticated = capable(CAP_SYS_ADMIN);
@@ -258,12 +231,11 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
 
         /* if there is no current master make this fd it, but do not create
          * any master object for render clients */
-        mutex_lock(&dev->struct_mutex);
-        if (!priv->minor->master && !drm_is_render_client(priv)) {
+        mutex_lock(&dev->master_mutex);
+        if (drm_is_primary_client(priv) && !priv->minor->master) {
                 /* create a new master */
                 priv->minor->master = drm_master_create(priv->minor);
                 if (!priv->minor->master) {
-                        mutex_unlock(&dev->struct_mutex);
                         ret = -ENOMEM;
                         goto out_close;
                 }
@@ -271,37 +243,31 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
                 priv->is_master = 1;
                 /* take another reference for the copy in the local file priv */
                 priv->master = drm_master_get(priv->minor->master);
-
                 priv->authenticated = 1;
 
-                mutex_unlock(&dev->struct_mutex);
                 if (dev->driver->master_create) {
                         ret = dev->driver->master_create(dev, priv->master);
                         if (ret) {
-                                mutex_lock(&dev->struct_mutex);
                                 /* drop both references if this fails */
                                 drm_master_put(&priv->minor->master);
                                 drm_master_put(&priv->master);
-                                mutex_unlock(&dev->struct_mutex);
                                 goto out_close;
                         }
                 }
-                mutex_lock(&dev->struct_mutex);
                 if (dev->driver->master_set) {
                         ret = dev->driver->master_set(dev, priv, true);
                         if (ret) {
                                 /* drop both references if this fails */
                                 drm_master_put(&priv->minor->master);
                                 drm_master_put(&priv->master);
-                                mutex_unlock(&dev->struct_mutex);
                                 goto out_close;
                         }
                 }
-        } else if (!drm_is_render_client(priv)) {
+        } else if (drm_is_primary_client(priv)) {
                 /* get a reference to the master */
                 priv->master = drm_master_get(priv->minor->master);
         }
-        mutex_unlock(&dev->struct_mutex);
+        mutex_unlock(&dev->master_mutex);
 
         mutex_lock(&dev->struct_mutex);
         list_add(&priv->lhead, &dev->filelist);
@@ -330,6 +296,7 @@ static int drm_open_helper(struct inode *inode, struct file *filp,
         return 0;
 
 out_close:
+        mutex_unlock(&dev->master_mutex);
         if (dev->driver->postclose)
                 dev->driver->postclose(dev, priv);
 out_prime_destroy:
@@ -337,7 +304,6 @@ out_prime_destroy:
                 drm_prime_destroy_file_private(&priv->prime);
         if (dev->driver->driver_features & DRIVER_GEM)
                 drm_gem_release(dev, priv);
-out_put_pid:
         put_pid(priv->pid);
         kfree(priv);
         filp->private_data = NULL;
@@ -435,7 +401,6 @@ int drm_lastclose(struct drm_device * dev)
 
         drm_legacy_dma_takedown(dev);
 
-        dev->dev_mapping = NULL;
         mutex_unlock(&dev->struct_mutex);
 
         drm_legacy_dev_reinit(dev);
@@ -459,7 +424,8 @@ int drm_lastclose(struct drm_device * dev)
 int drm_release(struct inode *inode, struct file *filp)
 {
         struct drm_file *file_priv = filp->private_data;
-        struct drm_device *dev = file_priv->minor->dev;
+        struct drm_minor *minor = file_priv->minor;
+        struct drm_device *dev = minor->dev;
         int retcode = 0;
 
         mutex_lock(&drm_global_mutex);
@@ -475,7 +441,7 @@ int drm_release(struct inode *inode, struct file *filp)
 
         DRM_DEBUG("pid = %d, device = 0x%lx, open_count = %d\n",
                   task_pid_nr(current),
-                  (long)old_encode_dev(file_priv->minor->device),
+                  (long)old_encode_dev(file_priv->minor->kdev->devt),
                   dev->open_count);
 
         /* Release any auth tokens that might point to this file_priv,
@@ -518,11 +484,13 @@ int drm_release(struct inode *inode, struct file *filp)
         }
         mutex_unlock(&dev->ctxlist_mutex);
 
-        mutex_lock(&dev->struct_mutex);
+        mutex_lock(&dev->master_mutex);
 
         if (file_priv->is_master) {
                 struct drm_master *master = file_priv->master;
                 struct drm_file *temp;
+
+                mutex_lock(&dev->struct_mutex);
                 list_for_each_entry(temp, &dev->filelist, lhead) {
                         if ((temp->master == file_priv->master) &&
                             (temp != file_priv))
@@ -541,6 +509,7 @@ int drm_release(struct inode *inode, struct file *filp)
                         master->lock.file_priv = NULL;
                         wake_up_interruptible_all(&master->lock.lock_queue);
                 }
+                mutex_unlock(&dev->struct_mutex);
 
                 if (file_priv->minor->master == file_priv->master) {
                         /* drop the reference held my the minor */
@@ -550,13 +519,13 @@ int drm_release(struct inode *inode, struct file *filp)
                 }
         }
 
-        BUG_ON(dev->dev_mapping == NULL);
-        iput(container_of(dev->dev_mapping, struct inode, i_data));
-
-        /* drop the reference held my the file priv */
+        /* drop the master reference held by the file priv */
         if (file_priv->master)
                 drm_master_put(&file_priv->master);
         file_priv->is_master = 0;
+        mutex_unlock(&dev->master_mutex);
+
+        mutex_lock(&dev->struct_mutex);
         list_del(&file_priv->lhead);
         mutex_unlock(&dev->struct_mutex);
 
@@ -581,6 +550,8 @@ int drm_release(struct inode *inode, struct file *filp)
         }
         mutex_unlock(&drm_global_mutex);
 
+        drm_minor_release(minor);
+
         return retcode;
 }
 EXPORT_SYMBOL(drm_release);
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index 5bbad873c798..9909bef59800 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -85,9 +85,9 @@
 #endif
 
 /**
- * Initialize the GEM device fields
+ * drm_gem_init - Initialize the GEM device fields
+ * @dev: drm_devic structure to initialize
  */
-
 int
 drm_gem_init(struct drm_device *dev)
 {
@@ -120,6 +120,11 @@ drm_gem_destroy(struct drm_device *dev)
 }
 
 /**
+ * drm_gem_object_init - initialize an allocated shmem-backed GEM object
+ * @dev: drm_device the object should be initialized for
+ * @obj: drm_gem_object to initialize
+ * @size: object size
+ *
  * Initialize an already allocated GEM object of the specified size with
  * shmfs backing store.
  */
@@ -141,6 +146,11 @@ int drm_gem_object_init(struct drm_device *dev,
 EXPORT_SYMBOL(drm_gem_object_init);
 
 /**
+ * drm_gem_object_init - initialize an allocated private GEM object
+ * @dev: drm_device the object should be initialized for
+ * @obj: drm_gem_object to initialize
+ * @size: object size
+ *
  * Initialize an already allocated GEM object of the specified size with
  * no GEM provided backing store. Instead the caller is responsible for
  * backing the object and handling it.
@@ -176,6 +186,9 @@ drm_gem_remove_prime_handles(struct drm_gem_object *obj, struct drm_file *filp)
 }
 
 /**
+ * drm_gem_object_free - release resources bound to userspace handles
+ * @obj: GEM object to clean up.
+ *
  * Called after the last handle to the object has been closed
  *
  * Removes any name for the object. Note that this must be
@@ -225,7 +238,12 @@ drm_gem_object_handle_unreference_unlocked(struct drm_gem_object *obj)
 }
 
 /**
- * Removes the mapping from handle to filp for this object.
+ * drm_gem_handle_delete - deletes the given file-private handle
+ * @filp: drm file-private structure to use for the handle look up
+ * @handle: userspace handle to delete
+ *
+ * Removes the GEM handle from the @filp lookup table and if this is the last
+ * handle also cleans up linked resources like GEM names.
  */
 int
 drm_gem_handle_delete(struct drm_file *filp, u32 handle)
@@ -270,6 +288,9 @@ EXPORT_SYMBOL(drm_gem_handle_delete);
 
 /**
  * drm_gem_dumb_destroy - dumb fb callback helper for gem based drivers
+ * @file: drm file-private structure to remove the dumb handle from
+ * @dev: corresponding drm_device
+ * @handle: the dumb handle to remove
  * 
  * This implements the ->dumb_destroy kms driver callback for drivers which use
  * gem to manage their backing storage.
@@ -284,6 +305,9 @@ EXPORT_SYMBOL(drm_gem_dumb_destroy);
 
 /**
  * drm_gem_handle_create_tail - internal functions to create a handle
+ * @file_priv: drm file-private structure to register the handle for
+ * @obj: object to register
+ * @handlep: pionter to return the created handle to the caller
  * 
  * This expects the dev->object_name_lock to be held already and will drop it
  * before returning. Used to avoid races in establishing new handles when
@@ -336,6 +360,11 @@ drm_gem_handle_create_tail(struct drm_file *file_priv,
 }
 
 /**
+ * gem_handle_create - create a gem handle for an object
+ * @file_priv: drm file-private structure to register the handle for
+ * @obj: object to register
+ * @handlep: pionter to return the created handle to the caller
+ *
  * Create a handle for this object. This adds a handle reference
  * to the object, which includes a regular reference count. Callers
  * will likely want to dereference the object afterwards.
@@ -536,6 +565,11 @@ drm_gem_object_lookup(struct drm_device *dev, struct drm_file *filp,
 EXPORT_SYMBOL(drm_gem_object_lookup);
 
 /**
+ * drm_gem_close_ioctl - implementation of the GEM_CLOSE ioctl
+ * @dev: drm_device
+ * @data: ioctl data
+ * @file_priv: drm file-private structure
+ *
  * Releases the handle to an mm object.
  */
 int
@@ -554,6 +588,11 @@ drm_gem_close_ioctl(struct drm_device *dev, void *data,
 }
 
 /**
+ * drm_gem_flink_ioctl - implementation of the GEM_FLINK ioctl
+ * @dev: drm_device
+ * @data: ioctl data
+ * @file_priv: drm file-private structure
+ *
  * Create a global name for an object, returning the name.
  *
  * Note that the name does not hold a reference; when the object
@@ -601,6 +640,11 @@ err:
 }
 
 /**
+ * drm_gem_open - implementation of the GEM_OPEN ioctl
+ * @dev: drm_device
+ * @data: ioctl data
+ * @file_priv: drm file-private structure
+ *
  * Open an object using the global name, returning a handle and the size.
  *
  * This handle (of course) holds a reference to the object, so the object
@@ -640,6 +684,10 @@ drm_gem_open_ioctl(struct drm_device *dev, void *data,
 }
 
 /**
+ * gem_gem_open - initalizes GEM file-private structures at devnode open time
+ * @dev: drm_device which is being opened by userspace
+ * @file_private: drm file-private structure to set up
+ *
  * Called at device open time, sets up the structure for handling refcounting
  * of mm objects.
  */
@@ -650,7 +698,7 @@ drm_gem_open(struct drm_device *dev, struct drm_file *file_private)
         spin_lock_init(&file_private->table_lock);
 }
 
-/**
+/*
  * Called at device close to release the file's
  * handle references on objects.
  */
@@ -674,6 +722,10 @@ drm_gem_object_release_handle(int id, void *ptr, void *data)
 }
 
 /**
+ * drm_gem_release - release file-private GEM resources
+ * @dev: drm_device which is being closed by userspace
+ * @file_private: drm file-private structure to clean up
+ *
  * Called at close time when the filp is going away.
  *
  * Releases any remaining references on objects by this filp.
@@ -692,11 +744,16 @@ drm_gem_object_release(struct drm_gem_object *obj)
         WARN_ON(obj->dma_buf);
 
         if (obj->filp)
-            fput(obj->filp);
+                fput(obj->filp);
+
+        drm_gem_free_mmap_offset(obj);
 }
 EXPORT_SYMBOL(drm_gem_object_release);
 
 /**
+ * drm_gem_object_free - free a GEM object
+ * @kref: kref of the object to free
+ *
  * Called after the last reference to the object has been lost.
  * Must be called holding struct_ mutex
  *
@@ -782,7 +839,7 @@ int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size,
         vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP;
         vma->vm_ops = dev->driver->gem_vm_ops;
         vma->vm_private_data = obj;
-        vma->vm_page_prot =  pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+        vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
 
         /* Take a ref for this mapping of the object, so that the fault
          * handler can dereference the mmap offset's pointer to the object.
@@ -818,7 +875,7 @@ int drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
         struct drm_device *dev = priv->minor->dev;
         struct drm_gem_object *obj;
         struct drm_vma_offset_node *node;
-        int ret = 0;
+        int ret;
 
         if (drm_device_is_unplugged(dev))
                 return -ENODEV;
diff --git a/drivers/gpu/drm/drm_gem_cma_helper.c b/drivers/gpu/drm/drm_gem_cma_helper.c
index 6b51bf90df0e..05c97c5350a1 100644
--- a/drivers/gpu/drm/drm_gem_cma_helper.c
+++ b/drivers/gpu/drm/drm_gem_cma_helper.c
@@ -79,7 +79,6 @@ struct drm_gem_cma_object *drm_gem_cma_create(struct drm_device *drm,
                 unsigned int size)
 {
         struct drm_gem_cma_object *cma_obj;
-        struct sg_table *sgt = NULL;
         int ret;
 
         size = round_up(size, PAGE_SIZE);
@@ -97,23 +96,9 @@ struct drm_gem_cma_object *drm_gem_cma_create(struct drm_device *drm,
                 goto error;
         }
 
-        sgt = kzalloc(sizeof(*cma_obj->sgt), GFP_KERNEL);
-        if (sgt == NULL) {
-                ret = -ENOMEM;
-                goto error;
-        }
-
-        ret = dma_get_sgtable(drm->dev, sgt, cma_obj->vaddr,
-                              cma_obj->paddr, size);
-        if (ret < 0)
-                goto error;
-
-        cma_obj->sgt = sgt;
-
         return cma_obj;
 
 error:
-        kfree(sgt);
         drm_gem_cma_free_object(&cma_obj->base);
         return ERR_PTR(ret);
 }
@@ -175,10 +160,6 @@ void drm_gem_cma_free_object(struct drm_gem_object *gem_obj)
         if (cma_obj->vaddr) {
                 dma_free_writecombine(gem_obj->dev->dev, cma_obj->base.size,
                                       cma_obj->vaddr, cma_obj->paddr);
-                if (cma_obj->sgt) {
-                        sg_free_table(cma_obj->sgt);
-                        kfree(cma_obj->sgt);
-                }
         } else if (gem_obj->import_attach) {
                 drm_prime_gem_destroy(gem_obj, cma_obj->sgt);
         }
@@ -253,8 +234,17 @@ static int drm_gem_cma_mmap_obj(struct drm_gem_cma_object *cma_obj,
 {
         int ret;
 
-        ret = remap_pfn_range(vma, vma->vm_start, cma_obj->paddr >> PAGE_SHIFT,
-                        vma->vm_end - vma->vm_start, vma->vm_page_prot);
+        /*
+         * Clear the VM_PFNMAP flag that was set by drm_gem_mmap(), and set the
+         * vm_pgoff (used as a fake buffer offset by DRM) to 0 as we want to map
+         * the whole buffer.
+         */
+        vma->vm_flags &= ~VM_PFNMAP;
+        vma->vm_pgoff = 0;
+
+        ret = dma_mmap_writecombine(cma_obj->base.dev->dev, vma,
+                                    cma_obj->vaddr, cma_obj->paddr,
+                                    vma->vm_end - vma->vm_start);
         if (ret)
                 drm_gem_vm_close(vma);
 
@@ -292,9 +282,9 @@ void drm_gem_cma_describe(struct drm_gem_cma_object *cma_obj, struct seq_file *m
 
         off = drm_vma_node_start(&obj->vma_node);
 
-        seq_printf(m, "%2d (%2d) %08llx %08Zx %p %d",
+        seq_printf(m, "%2d (%2d) %08llx %pad %p %d",
                         obj->name, obj->refcount.refcount.counter,
-                        off, cma_obj->paddr, cma_obj->vaddr, obj->size);
+                        off, &cma_obj->paddr, cma_obj->vaddr, obj->size);
 
         seq_printf(m, "\n");
 }
@@ -342,7 +332,7 @@ drm_gem_cma_prime_import_sg_table(struct drm_device *dev, size_t size,
         cma_obj->paddr = sg_dma_address(sgt->sgl);
         cma_obj->sgt = sgt;
 
-        DRM_DEBUG_PRIME("dma_addr = 0x%x, size = %zu\n", cma_obj->paddr, size);
+        DRM_DEBUG_PRIME("dma_addr = %pad, size = %zu\n", &cma_obj->paddr, size);
 
         return &cma_obj->base;
 }
diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index f4dc9b7a3831..93a42040bedb 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -328,6 +328,13 @@ drm_setclientcap(struct drm_device *dev, void *data, struct drm_file *file_priv)
                         return -EINVAL;
                 file_priv->stereo_allowed = req->value;
                 break;
+        case DRM_CLIENT_CAP_UNIVERSAL_PLANES:
+                if (!drm_universal_planes)
+                        return -EINVAL;
+                if (req->value > 1)
+                        return -EINVAL;
+                file_priv->universal_planes = req->value;
+                break;
         default:
                 return -EINVAL;
         }
diff --git a/drivers/gpu/drm/drm_mipi_dsi.c b/drivers/gpu/drm/drm_mipi_dsi.c
index b155ee2ffa17..09821f46d768 100644
--- a/drivers/gpu/drm/drm_mipi_dsi.c
+++ b/drivers/gpu/drm/drm_mipi_dsi.c
@@ -142,8 +142,12 @@ int mipi_dsi_host_register(struct mipi_dsi_host *host)
 {
         struct device_node *node;
 
-        for_each_available_child_of_node(host->dev->of_node, node)
+        for_each_available_child_of_node(host->dev->of_node, node) {
+                /* skip nodes without reg property */
+                if (!of_find_property(node, "reg", NULL))
+                        continue;
                 of_mipi_dsi_device_add(host, node);
+        }
 
         return 0;
 }
diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c
index af93cc55259f..71e2d3fcd6ee 100644
--- a/drivers/gpu/drm/drm_mm.c
+++ b/drivers/gpu/drm/drm_mm.c
@@ -47,7 +47,48 @@
 #include <linux/seq_file.h>
 #include <linux/export.h>
 
-#define MM_UNUSED_TARGET 4
+/**
+ * DOC: Overview
+ *
+ * drm_mm provides a simple range allocator. The drivers are free to use the
+ * resource allocator from the linux core if it suits them, the upside of drm_mm
+ * is that it's in the DRM core. Which means that it's easier to extend for
+ * some of the crazier special purpose needs of gpus.
+ *
+ * The main data struct is &drm_mm, allocations are tracked in &drm_mm_node.
+ * Drivers are free to embed either of them into their own suitable
+ * datastructures. drm_mm itself will not do any allocations of its own, so if
+ * drivers choose not to embed nodes they need to still allocate them
+ * themselves.
+ *
+ * The range allocator also supports reservation of preallocated blocks. This is
+ * useful for taking over initial mode setting configurations from the firmware,
+ * where an object needs to be created which exactly matches the firmware's
+ * scanout target. As long as the range is still free it can be inserted anytime
+ * after the allocator is initialized, which helps with avoiding looped
+ * depencies in the driver load sequence.
+ *
+ * drm_mm maintains a stack of most recently freed holes, which of all
+ * simplistic datastructures seems to be a fairly decent approach to clustering
+ * allocations and avoiding too much fragmentation. This means free space
+ * searches are O(num_holes). Given that all the fancy features drm_mm supports
+ * something better would be fairly complex and since gfx thrashing is a fairly
+ * steep cliff not a real concern. Removing a node again is O(1).
+ *
+ * drm_mm supports a few features: Alignment and range restrictions can be
+ * supplied. Further more every &drm_mm_node has a color value (which is just an
+ * opaqua unsigned long) which in conjunction with a driver callback can be used
+ * to implement sophisticated placement restrictions. The i915 DRM driver uses
+ * this to implement guard pages between incompatible caching domains in the
+ * graphics TT.
+ *
+ * Two behaviors are supported for searching and allocating: bottom-up and top-down.
+ * The default is bottom-up. Top-down allocation can be used if the memory area
+ * has different restrictions, or just to reduce fragmentation.
+ *
+ * Finally iteration helpers to walk all nodes and all holes are provided as are
+ * some basic allocator dumpers for debugging.
+ */
 
 static struct drm_mm_node *drm_mm_search_free_generic(const struct drm_mm *mm,
                                                 unsigned long size,
@@ -65,7 +106,8 @@ static struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_
 static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
                                  struct drm_mm_node *node,
                                  unsigned long size, unsigned alignment,
-                                 unsigned long color)
+                                 unsigned long color,
+                                 enum drm_mm_allocator_flags flags)
 {
         struct drm_mm *mm = hole_node->mm;
         unsigned long hole_start = drm_mm_hole_node_start(hole_node);
@@ -78,12 +120,22 @@ static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
         if (mm->color_adjust)
                 mm->color_adjust(hole_node, color, &adj_start, &adj_end);
 
+        if (flags & DRM_MM_CREATE_TOP)
+                adj_start = adj_end - size;
+
         if (alignment) {
                 unsigned tmp = adj_start % alignment;
-                if (tmp)
-                        adj_start += alignment - tmp;
+                if (tmp) {
+                        if (flags & DRM_MM_CREATE_TOP)
+                                adj_start -= tmp;
+                        else
+                                adj_start += alignment - tmp;
+                }
         }
 
+        BUG_ON(adj_start < hole_start);
+        BUG_ON(adj_end > hole_end);
+
         if (adj_start == hole_start) {
                 hole_node->hole_follows = 0;
                 list_del(&hole_node->hole_stack);
@@ -107,6 +159,20 @@ static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
         }
 }
 
+/**
+ * drm_mm_reserve_node - insert an pre-initialized node
+ * @mm: drm_mm allocator to insert @node into
+ * @node: drm_mm_node to insert
+ *
+ * This functions inserts an already set-up drm_mm_node into the allocator,
+ * meaning that start, size and color must be set by the caller. This is useful
+ * to initialize the allocator with preallocated objects which must be set-up
+ * before the range allocator can be set-up, e.g. when taking over a firmware
+ * framebuffer.
+ *
+ * Returns:
+ * 0 on success, -ENOSPC if there's no hole where @node is.
+ */
 int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node)
 {
         struct drm_mm_node *hole;
@@ -148,23 +214,34 @@ int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node)
 EXPORT_SYMBOL(drm_mm_reserve_node);
 
 /**
- * Search for free space and insert a preallocated memory node. Returns
- * -ENOSPC if no suitable free area is available. The preallocated memory node
- * must be cleared.
+ * drm_mm_insert_node_generic - search for space and insert @node
+ * @mm: drm_mm to allocate from
+ * @node: preallocate node to insert
+ * @size: size of the allocation
+ * @alignment: alignment of the allocation
+ * @color: opaque tag value to use for this node
+ * @sflags: flags to fine-tune the allocation search
+ * @aflags: flags to fine-tune the allocation behavior
+ *
+ * The preallocated node must be cleared to 0.
+ *
+ * Returns:
+ * 0 on success, -ENOSPC if there's no suitable hole.
  */
 int drm_mm_insert_node_generic(struct drm_mm *mm, struct drm_mm_node *node,
                                unsigned long size, unsigned alignment,
                                unsigned long color,
-                               enum drm_mm_search_flags flags)
+                               enum drm_mm_search_flags sflags,
+                               enum drm_mm_allocator_flags aflags)
 {
         struct drm_mm_node *hole_node;
 
         hole_node = drm_mm_search_free_generic(mm, size, alignment,
-                                               color, flags);
+                                               color, sflags);
         if (!hole_node)
                 return -ENOSPC;
 
-        drm_mm_insert_helper(hole_node, node, size, alignment, color);
+        drm_mm_insert_helper(hole_node, node, size, alignment, color, aflags);
         return 0;
 }
 EXPORT_SYMBOL(drm_mm_insert_node_generic);
@@ -173,7 +250,8 @@ static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node,
                                        struct drm_mm_node *node,
                                        unsigned long size, unsigned alignment,
                                        unsigned long color,
-                                       unsigned long start, unsigned long end)
+                                       unsigned long start, unsigned long end,
+                                       enum drm_mm_allocator_flags flags)
 {
         struct drm_mm *mm = hole_node->mm;
         unsigned long hole_start = drm_mm_hole_node_start(hole_node);
@@ -188,13 +266,20 @@ static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node,
         if (adj_end > end)
                 adj_end = end;
 
+        if (flags & DRM_MM_CREATE_TOP)
+                adj_start = adj_end - size;
+
         if (mm->color_adjust)
                 mm->color_adjust(hole_node, color, &adj_start, &adj_end);
 
         if (alignment) {
                 unsigned tmp = adj_start % alignment;
-                if (tmp)
-                        adj_start += alignment - tmp;
+                if (tmp) {
+                        if (flags & DRM_MM_CREATE_TOP)
+                                adj_start -= tmp;
+                        else
+                                adj_start += alignment - tmp;
+                }
         }
 
         if (adj_start == hole_start) {
@@ -211,6 +296,8 @@ static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node,
         INIT_LIST_HEAD(&node->hole_stack);
         list_add(&node->node_list, &hole_node->node_list);
 
+        BUG_ON(node->start < start);
+        BUG_ON(node->start < adj_start);
         BUG_ON(node->start + node->size > adj_end);
         BUG_ON(node->start + node->size > end);
 
@@ -222,32 +309,51 @@ static void drm_mm_insert_helper_range(struct drm_mm_node *hole_node,
 }
 
 /**
- * Search for free space and insert a preallocated memory node. Returns
- * -ENOSPC if no suitable free area is available. This is for range
- * restricted allocations. The preallocated memory node must be cleared.
+ * drm_mm_insert_node_in_range_generic - ranged search for space and insert @node
+ * @mm: drm_mm to allocate from
+ * @node: preallocate node to insert
+ * @size: size of the allocation
+ * @alignment: alignment of the allocation
+ * @color: opaque tag value to use for this node
+ * @start: start of the allowed range for this node
+ * @end: end of the allowed range for this node
+ * @sflags: flags to fine-tune the allocation search
+ * @aflags: flags to fine-tune the allocation behavior
+ *
+ * The preallocated node must be cleared to 0.
+ *
+ * Returns:
+ * 0 on success, -ENOSPC if there's no suitable hole.
  */
 int drm_mm_insert_node_in_range_generic(struct drm_mm *mm, struct drm_mm_node *node,
-                                        unsigned long size, unsigned alignment, unsigned long color,
+                                        unsigned long size, unsigned alignment,
+                                        unsigned long color,
                                         unsigned long start, unsigned long end,
-                                        enum drm_mm_search_flags flags)
+                                        enum drm_mm_search_flags sflags,
+                                        enum drm_mm_allocator_flags aflags)
 {
         struct drm_mm_node *hole_node;
 
         hole_node = drm_mm_search_free_in_range_generic(mm,
                                                         size, alignment, color,
-                                                        start, end, flags);
+                                                        start, end, sflags);
         if (!hole_node)
                 return -ENOSPC;
 
         drm_mm_insert_helper_range(hole_node, node,
                                    size, alignment, color,
-                                   start, end);
+                                   start, end, aflags);
         return 0;
 }
 EXPORT_SYMBOL(drm_mm_insert_node_in_range_generic);
 
 /**
- * Remove a memory node from the allocator.
+ * drm_mm_remove_node - Remove a memory node from the allocator.
+ * @node: drm_mm_node to remove
+ *
+ * This just removes a node from its drm_mm allocator. The node does not need to
+ * be cleared again before it can be re-inserted into this or any other drm_mm
+ * allocator. It is a bug to call this function on a un-allocated node.
  */
 void drm_mm_remove_node(struct drm_mm_node *node)
 {
@@ -315,7 +421,10 @@ static struct drm_mm_node *drm_mm_search_free_generic(const struct drm_mm *mm,
         best = NULL;
         best_size = ~0UL;
 
-        drm_mm_for_each_hole(entry, mm, adj_start, adj_end) {
+        __drm_mm_for_each_hole(entry, mm, adj_start, adj_end,
+                               flags & DRM_MM_SEARCH_BELOW) {
+                unsigned long hole_size = adj_end - adj_start;
+
                 if (mm->color_adjust) {
                         mm->color_adjust(entry, color, &adj_start, &adj_end);
                         if (adj_end <= adj_start)
@@ -328,9 +437,9 @@ static struct drm_mm_node *drm_mm_search_free_generic(const struct drm_mm *mm,
                 if (!(flags & DRM_MM_SEARCH_BEST))
                         return entry;
 
-                if (entry->size < best_size) {
+                if (hole_size < best_size) {
                         best = entry;
-                        best_size = entry->size;
+                        best_size = hole_size;
                 }
         }
 
@@ -356,7 +465,10 @@ static struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_
         best = NULL;
         best_size = ~0UL;
 
-        drm_mm_for_each_hole(entry, mm, adj_start, adj_end) {
+        __drm_mm_for_each_hole(entry, mm, adj_start, adj_end,
+                               flags & DRM_MM_SEARCH_BELOW) {
+                unsigned long hole_size = adj_end - adj_start;
+
                 if (adj_start < start)
                         adj_start = start;
                 if (adj_end > end)
@@ -374,9 +486,9 @@ static struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_
                 if (!(flags & DRM_MM_SEARCH_BEST))
                         return entry;
 
-                if (entry->size < best_size) {
+                if (hole_size < best_size) {
                         best = entry;
-                        best_size = entry->size;
+                        best_size = hole_size;
                 }
         }
 
@@ -384,7 +496,13 @@ static struct drm_mm_node *drm_mm_search_free_in_range_generic(const struct drm_
 }
 
 /**
- * Moves an allocation. To be used with embedded struct drm_mm_node.
+ * drm_mm_replace_node - move an allocation from @old to @new
+ * @old: drm_mm_node to remove from the allocator
+ * @new: drm_mm_node which should inherit @old's allocation
+ *
+ * This is useful for when drivers embed the drm_mm_node structure and hence
+ * can't move allocations by reassigning pointers. It's a combination of remove
+ * and insert with the guarantee that the allocation start will match.
  */
 void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new)
 {
@@ -402,12 +520,46 @@ void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new)
 EXPORT_SYMBOL(drm_mm_replace_node);
 
 /**
- * Initializa lru scanning.
+ * DOC: lru scan roaster
+ *
+ * Very often GPUs need to have continuous allocations for a given object. When
+ * evicting objects to make space for a new one it is therefore not most
+ * efficient when we simply start to select all objects from the tail of an LRU
+ * until there's a suitable hole: Especially for big objects or nodes that
+ * otherwise have special allocation constraints there's a good chance we evict
+ * lots of (smaller) objects unecessarily.
+ *
+ * The DRM range allocator supports this use-case through the scanning
+ * interfaces. First a scan operation needs to be initialized with
+ * drm_mm_init_scan() or drm_mm_init_scan_with_range(). The the driver adds
+ * objects to the roaster (probably by walking an LRU list, but this can be
+ * freely implemented) until a suitable hole is found or there's no further
+ * evitable object.
+ *
+ * The the driver must walk through all objects again in exactly the reverse
+ * order to restore the allocator state. Note that while the allocator is used
+ * in the scan mode no other operation is allowed.
+ *
+ * Finally the driver evicts all objects selected in the scan. Adding and
+ * removing an object is O(1), and since freeing a node is also O(1) the overall
+ * complexity is O(scanned_objects). So like the free stack which needs to be
+ * walked before a scan operation even begins this is linear in the number of
+ * objects. It doesn't seem to hurt badly.
+ */
+
+/**
+ * drm_mm_init_scan - initialize lru scanning
+ * @mm: drm_mm to scan
+ * @size: size of the allocation
+ * @alignment: alignment of the allocation
+ * @color: opaque tag value to use for the allocation
  *
  * This simply sets up the scanning routines with the parameters for the desired
- * hole.
+ * hole. Note that there's no need to specify allocation flags, since they only
+ * change the place a node is allocated from within a suitable hole.
  *
- * Warning: As long as the scan list is non-empty, no other operations than
+ * Warning:
+ * As long as the scan list is non-empty, no other operations than
  * adding/removing nodes to/from the scan list are allowed.
  */
 void drm_mm_init_scan(struct drm_mm *mm,
@@ -427,12 +579,20 @@ void drm_mm_init_scan(struct drm_mm *mm,
 EXPORT_SYMBOL(drm_mm_init_scan);
 
 /**
- * Initializa lru scanning.
+ * drm_mm_init_scan - initialize range-restricted lru scanning
+ * @mm: drm_mm to scan
+ * @size: size of the allocation
+ * @alignment: alignment of the allocation
+ * @color: opaque tag value to use for the allocation
+ * @start: start of the allowed range for the allocation
+ * @end: end of the allowed range for the allocation
  *
  * This simply sets up the scanning routines with the parameters for the desired
- * hole. This version is for range-restricted scans.
+ * hole. Note that there's no need to specify allocation flags, since they only
+ * change the place a node is allocated from within a suitable hole.
  *
- * Warning: As long as the scan list is non-empty, no other operations than
+ * Warning:
+ * As long as the scan list is non-empty, no other operations than
  * adding/removing nodes to/from the scan list are allowed.
  */
 void drm_mm_init_scan_with_range(struct drm_mm *mm,
@@ -456,12 +616,16 @@ void drm_mm_init_scan_with_range(struct drm_mm *mm,
 EXPORT_SYMBOL(drm_mm_init_scan_with_range);
 
 /**
+ * drm_mm_scan_add_block - add a node to the scan list
+ * @node: drm_mm_node to add
+ *
  * Add a node to the scan list that might be freed to make space for the desired
  * hole.
  *
- * Returns non-zero, if a hole has been found, zero otherwise.
+ * Returns:
+ * True if a hole has been found, false otherwise.
  */
-int drm_mm_scan_add_block(struct drm_mm_node *node)
+bool drm_mm_scan_add_block(struct drm_mm_node *node)
 {
         struct drm_mm *mm = node->mm;
         struct drm_mm_node *prev_node;
@@ -501,15 +665,16 @@ int drm_mm_scan_add_block(struct drm_mm_node *node)
                             mm->scan_size, mm->scan_alignment)) {
                 mm->scan_hit_start = hole_start;
                 mm->scan_hit_end = hole_end;
-                return 1;
+                return true;
         }
 
-        return 0;
+        return false;
 }
 EXPORT_SYMBOL(drm_mm_scan_add_block);
 
 /**
- * Remove a node from the scan list.
+ * drm_mm_scan_remove_block - remove a node from the scan list
+ * @node: drm_mm_node to remove
  *
  * Nodes _must_ be removed in the exact same order from the scan list as they
  * have been added, otherwise the internal state of the memory manager will be
@@ -519,10 +684,11 @@ EXPORT_SYMBOL(drm_mm_scan_add_block);
  * immediately following drm_mm_search_free with !DRM_MM_SEARCH_BEST will then
  * return the just freed block (because its at the top of the free_stack list).
  *
- * Returns one if this block should be evicted, zero otherwise. Will always
- * return zero when no hole has been found.
+ * Returns:
+ * True if this block should be evicted, false otherwise. Will always
+ * return false when no hole has been found.
  */
-int drm_mm_scan_remove_block(struct drm_mm_node *node)
+bool drm_mm_scan_remove_block(struct drm_mm_node *node)
 {
         struct drm_mm *mm = node->mm;
         struct drm_mm_node *prev_node;
@@ -543,7 +709,15 @@ int drm_mm_scan_remove_block(struct drm_mm_node *node)
 }
 EXPORT_SYMBOL(drm_mm_scan_remove_block);
 
-int drm_mm_clean(struct drm_mm * mm)
+/**
+ * drm_mm_clean - checks whether an allocator is clean
+ * @mm: drm_mm allocator to check
+ *
+ * Returns:
+ * True if the allocator is completely free, false if there's still a node
+ * allocated in it.
+ */
+bool drm_mm_clean(struct drm_mm * mm)
 {
         struct list_head *head = &mm->head_node.node_list;
 
@@ -551,6 +725,14 @@ int drm_mm_clean(struct drm_mm * mm)
 }
 EXPORT_SYMBOL(drm_mm_clean);
 
+/**
+ * drm_mm_init - initialize a drm-mm allocator
+ * @mm: the drm_mm structure to initialize
+ * @start: start of the range managed by @mm
+ * @size: end of the range managed by @mm
+ *
+ * Note that @mm must be cleared to 0 before calling this function.
+ */
 void drm_mm_init(struct drm_mm * mm, unsigned long start, unsigned long size)
 {
         INIT_LIST_HEAD(&mm->hole_stack);
@@ -572,6 +754,13 @@ void drm_mm_init(struct drm_mm * mm, unsigned long start, unsigned long size)
 }
 EXPORT_SYMBOL(drm_mm_init);
 
+/**
+ * drm_mm_takedown - clean up a drm_mm allocator
+ * @mm: drm_mm allocator to clean up
+ *
+ * Note that it is a bug to call this function on an allocator which is not
+ * clean.
+ */
 void drm_mm_takedown(struct drm_mm * mm)
 {
         WARN(!list_empty(&mm->head_node.node_list),
@@ -597,6 +786,11 @@ static unsigned long drm_mm_debug_hole(struct drm_mm_node *entry,
         return 0;
 }
 
+/**
+ * drm_mm_debug_table - dump allocator state to dmesg
+ * @mm: drm_mm allocator to dump
+ * @prefix: prefix to use for dumping to dmesg
+ */
 void drm_mm_debug_table(struct drm_mm *mm, const char *prefix)
 {
         struct drm_mm_node *entry;
@@ -635,6 +829,11 @@ static unsigned long drm_mm_dump_hole(struct seq_file *m, struct drm_mm_node *en
         return 0;
 }
 
+/**
+ * drm_mm_dump_table - dump allocator state to a seq_file
+ * @m: seq_file to dump to
+ * @mm: drm_mm allocator to dump
+ */
 int drm_mm_dump_table(struct seq_file *m, struct drm_mm *mm)
 {
         struct drm_mm_node *entry;
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index b0733153dfd2..8b410576fce4 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -37,15 +37,14 @@
 #include <drm/drm_crtc.h>
 #include <video/of_videomode.h>
 #include <video/videomode.h>
+#include <drm/drm_modes.h>
+
+#include "drm_crtc_internal.h"
 
 /**
- * drm_mode_debug_printmodeline - debug print a mode
- * @dev: DRM device
+ * drm_mode_debug_printmodeline - print a mode to dmesg
  * @mode: mode to print
  *
- * LOCKING:
- * None.
- *
  * Describe @mode using DRM_DEBUG.
  */
 void drm_mode_debug_printmodeline(const struct drm_display_mode *mode)
@@ -61,18 +60,77 @@ void drm_mode_debug_printmodeline(const struct drm_display_mode *mode)
 EXPORT_SYMBOL(drm_mode_debug_printmodeline);
 
 /**
- * drm_cvt_mode -create a modeline based on CVT algorithm
+ * drm_mode_create - create a new display mode
  * @dev: DRM device
- * @hdisplay: hdisplay size
- * @vdisplay: vdisplay size
- * @vrefresh  : vrefresh rate
- * @reduced : Whether the GTF calculation is simplified
- * @interlaced:Whether the interlace is supported
  *
- * LOCKING:
- * none.
+ * Create a new, cleared drm_display_mode with kzalloc, allocate an ID for it
+ * and return it.
  *
- * return the modeline based on CVT algorithm
+ * Returns:
+ * Pointer to new mode on success, NULL on error.
+ */
+struct drm_display_mode *drm_mode_create(struct drm_device *dev)
+{
+        struct drm_display_mode *nmode;
+
+        nmode = kzalloc(sizeof(struct drm_display_mode), GFP_KERNEL);
+        if (!nmode)
+                return NULL;
+
+        if (drm_mode_object_get(dev, &nmode->base, DRM_MODE_OBJECT_MODE)) {
+                kfree(nmode);
+                return NULL;
+        }
+
+        return nmode;
+}
+EXPORT_SYMBOL(drm_mode_create);
+
+/**
+ * drm_mode_destroy - remove a mode
+ * @dev: DRM device
+ * @mode: mode to remove
+ *
+ * Release @mode's unique ID, then free it @mode structure itself using kfree.
+ */
+void drm_mode_destroy(struct drm_device *dev, struct drm_display_mode *mode)
+{
+        if (!mode)
+                return;
+
+        drm_mode_object_put(dev, &mode->base);
+
+        kfree(mode);
+}
+EXPORT_SYMBOL(drm_mode_destroy);
+
+/**
+ * drm_mode_probed_add - add a mode to a connector's probed_mode list
+ * @connector: connector the new mode
+ * @mode: mode data
+ *
+ * Add @mode to @connector's probed_mode list for later use. This list should
+ * then in a second step get filtered and all the modes actually supported by
+ * the hardware moved to the @connector's modes list.
+ */
+void drm_mode_probed_add(struct drm_connector *connector,
+                         struct drm_display_mode *mode)
+{
+        WARN_ON(!mutex_is_locked(&connector->dev->mode_config.mutex));
+
+        list_add_tail(&mode->head, &connector->probed_modes);
+}
+EXPORT_SYMBOL(drm_mode_probed_add);
+
+/**
+ * drm_cvt_mode -create a modeline based on the CVT algorithm
+ * @dev: drm device
+ * @hdisplay: hdisplay size
+ * @vdisplay: vdisplay size
+ * @vrefresh: vrefresh rate
+ * @reduced: whether to use reduced blanking
+ * @interlaced: whether to compute an interlaced mode
+ * @margins: whether to add margins (borders)
  *
  * This function is called to generate the modeline based on CVT algorithm
  * according to the hdisplay, vdisplay, vrefresh.
@@ -82,12 +140,17 @@ EXPORT_SYMBOL(drm_mode_debug_printmodeline);
  *
  * And it is copied from xf86CVTmode in xserver/hw/xfree86/modes/xf86cvt.c.
  * What I have done is to translate it by using integer calculation.
+ *
+ * Returns:
+ * The modeline based on the CVT algorithm stored in a drm_display_mode object.
+ * The display mode object is allocated with drm_mode_create(). Returns NULL
+ * when no mode could be allocated.
  */
-#define HV_FACTOR                       1000
 struct drm_display_mode *drm_cvt_mode(struct drm_device *dev, int hdisplay,
                                       int vdisplay, int vrefresh,
                                       bool reduced, bool interlaced, bool margins)
 {
+#define HV_FACTOR                       1000
         /* 1) top/bottom margin size (% of height) - default: 1.8, */
 #define CVT_MARGIN_PERCENTAGE           18
         /* 2) character cell horizontal granularity (pixels) - default 8 */
@@ -281,23 +344,25 @@ struct drm_display_mode *drm_cvt_mode(struct drm_device *dev, int hdisplay,
 EXPORT_SYMBOL(drm_cvt_mode);
 
 /**
- * drm_gtf_mode_complex - create the modeline based on full GTF algorithm
- *
- * @dev         :drm device
- * @hdisplay    :hdisplay size
- * @vdisplay    :vdisplay size
- * @vrefresh    :vrefresh rate.
- * @interlaced  :whether the interlace is supported
- * @margins     :desired margin size
- * @GTF_[MCKJ]  :extended GTF formula parameters
- *
- * LOCKING.
- * none.
- *
- * return the modeline based on full GTF algorithm.
+ * drm_gtf_mode_complex - create the modeline based on the full GTF algorithm
+ * @dev: drm device
+ * @hdisplay: hdisplay size
+ * @vdisplay: vdisplay size
+ * @vrefresh: vrefresh rate.
+ * @interlaced: whether to compute an interlaced mode
+ * @margins: desired margin (borders) size
+ * @GTF_M: extended GTF formula parameters
+ * @GTF_2C: extended GTF formula parameters
+ * @GTF_K: extended GTF formula parameters
+ * @GTF_2J: extended GTF formula parameters
  *
  * GTF feature blocks specify C and J in multiples of 0.5, so we pass them
  * in here multiplied by two.  For a C of 40, pass in 80.
+ *
+ * Returns:
+ * The modeline based on the full GTF algorithm stored in a drm_display_mode object.
+ * The display mode object is allocated with drm_mode_create(). Returns NULL
+ * when no mode could be allocated.
  */
 struct drm_display_mode *
 drm_gtf_mode_complex(struct drm_device *dev, int hdisplay, int vdisplay,
@@ -467,17 +532,13 @@ drm_gtf_mode_complex(struct drm_device *dev, int hdisplay, int vdisplay,
 EXPORT_SYMBOL(drm_gtf_mode_complex);
 
 /**
- * drm_gtf_mode - create the modeline based on GTF algorithm
- *
- * @dev         :drm device
- * @hdisplay    :hdisplay size
- * @vdisplay    :vdisplay size
- * @vrefresh    :vrefresh rate.
- * @interlaced  :whether the interlace is supported
- * @margins     :whether the margin is supported
- *
- * LOCKING.
- * none.
+ * drm_gtf_mode - create the modeline based on the GTF algorithm
+ * @dev: drm device
+ * @hdisplay: hdisplay size
+ * @vdisplay: vdisplay size
+ * @vrefresh: vrefresh rate.
+ * @interlaced: whether to compute an interlaced mode
+ * @margins: desired margin (borders) size
  *
  * return the modeline based on GTF algorithm
  *
@@ -496,19 +557,32 @@ EXPORT_SYMBOL(drm_gtf_mode_complex);
  * C = 40
  * K = 128
  * J = 20
+ *
+ * Returns:
+ * The modeline based on the GTF algorithm stored in a drm_display_mode object.
+ * The display mode object is allocated with drm_mode_create(). Returns NULL
+ * when no mode could be allocated.
  */
 struct drm_display_mode *
 drm_gtf_mode(struct drm_device *dev, int hdisplay, int vdisplay, int vrefresh,
-             bool lace, int margins)
+             bool interlaced, int margins)
 {
-        return drm_gtf_mode_complex(dev, hdisplay, vdisplay, vrefresh, lace,
-                                    margins, 600, 40 * 2, 128, 20 * 2);
+        return drm_gtf_mode_complex(dev, hdisplay, vdisplay, vrefresh,
+                                    interlaced, margins,
+                                    600, 40 * 2, 128, 20 * 2);
 }
 EXPORT_SYMBOL(drm_gtf_mode);
 
 #ifdef CONFIG_VIDEOMODE_HELPERS
-int drm_display_mode_from_videomode(const struct videomode *vm,
-                                    struct drm_display_mode *dmode)
+/**
+ * drm_display_mode_from_videomode - fill in @dmode using @vm,
+ * @vm: videomode structure to use as source
+ * @dmode: drm_display_mode structure to use as destination
+ *
+ * Fills out @dmode using the display mode specified in @vm.
+ */
+void drm_display_mode_from_videomode(const struct videomode *vm,
+                                     struct drm_display_mode *dmode)
 {
         dmode->hdisplay = vm->hactive;
         dmode->hsync_start = dmode->hdisplay + vm->hfront_porch;
@@ -538,8 +612,6 @@ int drm_display_mode_from_videomode(const struct videomode *vm,
         if (vm->flags & DISPLAY_FLAGS_DOUBLECLK)
                 dmode->flags |= DRM_MODE_FLAG_DBLCLK;
         drm_mode_set_name(dmode);
-
-        return 0;
 }
 EXPORT_SYMBOL_GPL(drm_display_mode_from_videomode);
 
@@ -553,6 +625,9 @@ EXPORT_SYMBOL_GPL(drm_display_mode_from_videomode);
  * This function is expensive and should only be used, if only one mode is to be
  * read from DT. To get multiple modes start with of_get_display_timings and
  * work with that instead.
+ *
+ * Returns:
+ * 0 on success, a negative errno code when no of videomode node was found.
  */
 int of_get_drm_display_mode(struct device_node *np,
                             struct drm_display_mode *dmode, int index)
@@ -580,10 +655,8 @@ EXPORT_SYMBOL_GPL(of_get_drm_display_mode);
  * drm_mode_set_name - set the name on a mode
  * @mode: name will be set in this mode
  *
- * LOCKING:
- * None.
- *
- * Set the name of @mode to a standard format.
+ * Set the name of @mode to a standard format which is <hdisplay>x<vdisplay>
+ * with an optional 'i' suffix for interlaced modes.
  */
 void drm_mode_set_name(struct drm_display_mode *mode)
 {
@@ -595,54 +668,12 @@ void drm_mode_set_name(struct drm_display_mode *mode)
 }
 EXPORT_SYMBOL(drm_mode_set_name);
 
-/**
- * drm_mode_width - get the width of a mode
- * @mode: mode
- *
- * LOCKING:
- * None.
- *
- * Return @mode's width (hdisplay) value.
- *
- * FIXME: is this needed?
- *
- * RETURNS:
- * @mode->hdisplay
- */
-int drm_mode_width(const struct drm_display_mode *mode)
-{
-        return mode->hdisplay;
-
-}
-EXPORT_SYMBOL(drm_mode_width);
-
-/**
- * drm_mode_height - get the height of a mode
- * @mode: mode
- *
- * LOCKING:
- * None.
- *
- * Return @mode's height (vdisplay) value.
- *
- * FIXME: is this needed?
- *
- * RETURNS:
- * @mode->vdisplay
- */
-int drm_mode_height(const struct drm_display_mode *mode)
-{
-        return mode->vdisplay;
-}
-EXPORT_SYMBOL(drm_mode_height);
-
 /** drm_mode_hsync - get the hsync of a mode
  * @mode: mode
  *
- * LOCKING:
- * None.
- *
- * Return @modes's hsync rate in kHz, rounded to the nearest int.
+ * Returns:
+ * @modes's hsync rate in kHz, rounded to the nearest integer. Calculates the
+ * value first if it is not yet set.
  */
 int drm_mode_hsync(const struct drm_display_mode *mode)
 {
@@ -666,17 +697,9 @@ EXPORT_SYMBOL(drm_mode_hsync);
  * drm_mode_vrefresh - get the vrefresh of a mode
  * @mode: mode
  *
- * LOCKING:
- * None.
- *
- * Return @mode's vrefresh rate in Hz or calculate it if necessary.
- *
- * FIXME: why is this needed?  shouldn't vrefresh be set already?
- *
- * RETURNS:
- * Vertical refresh rate. It will be the result of actual value plus 0.5.
- * If it is 70.288, it will return 70Hz.
- * If it is 59.6, it will return 60Hz.
+ * Returns:
+ * @modes's vrefresh rate in Hz, rounded to the nearest integer. Calculates the
+ * value first if it is not yet set.
  */
 int drm_mode_vrefresh(const struct drm_display_mode *mode)
 {
@@ -705,14 +728,11 @@ int drm_mode_vrefresh(const struct drm_display_mode *mode)
 EXPORT_SYMBOL(drm_mode_vrefresh);
 
 /**
- * drm_mode_set_crtcinfo - set CRTC modesetting parameters
+ * drm_mode_set_crtcinfo - set CRTC modesetting timing parameters
  * @p: mode
  * @adjust_flags: a combination of adjustment flags
  *
- * LOCKING:
- * None.
- *
- * Setup the CRTC modesetting parameters for @p, adjusting if necessary.
+ * Setup the CRTC modesetting timing parameters for @p, adjusting if necessary.
  *
  * - The CRTC_INTERLACE_HALVE_V flag can be used to halve vertical timings of
  *   interlaced modes.
@@ -780,15 +800,11 @@ void drm_mode_set_crtcinfo(struct drm_display_mode *p, int adjust_flags)
 }
 EXPORT_SYMBOL(drm_mode_set_crtcinfo);
 
-
 /**
  * drm_mode_copy - copy the mode
  * @dst: mode to overwrite
  * @src: mode to copy
  *
- * LOCKING:
- * None.
- *
  * Copy an existing mode into another mode, preserving the object id and
  * list head of the destination mode.
  */
@@ -805,13 +821,14 @@ EXPORT_SYMBOL(drm_mode_copy);
 
 /**
  * drm_mode_duplicate - allocate and duplicate an existing mode
- * @m: mode to duplicate
- *
- * LOCKING:
- * None.
+ * @dev: drm_device to allocate the duplicated mode for
+ * @mode: mode to duplicate
  *
  * Just allocate a new mode, copy the existing mode into it, and return
  * a pointer to it.  Used to create new instances of established modes.
+ *
+ * Returns:
+ * Pointer to duplicated mode on success, NULL on error.
  */
 struct drm_display_mode *drm_mode_duplicate(struct drm_device *dev,
                                             const struct drm_display_mode *mode)
@@ -833,12 +850,9 @@ EXPORT_SYMBOL(drm_mode_duplicate);
  * @mode1: first mode
  * @mode2: second mode
  *
- * LOCKING:
- * None.
- *
  * Check to see if @mode1 and @mode2 are equivalent.
  *
- * RETURNS:
+ * Returns:
  * True if the modes are equal, false otherwise.
  */
 bool drm_mode_equal(const struct drm_display_mode *mode1, const struct drm_display_mode *mode2)
@@ -864,13 +878,10 @@ EXPORT_SYMBOL(drm_mode_equal);
  * @mode1: first mode
  * @mode2: second mode
  *
- * LOCKING:
- * None.
- *
  * Check to see if @mode1 and @mode2 are equivalent, but
  * don't check the pixel clocks nor the stereo layout.
  *
- * RETURNS:
+ * Returns:
  * True if the modes are equal, false otherwise.
  */
 bool drm_mode_equal_no_clocks_no_stereo(const struct drm_display_mode *mode1,
@@ -900,25 +911,19 @@ EXPORT_SYMBOL(drm_mode_equal_no_clocks_no_stereo);
  * @mode_list: list of modes to check
  * @maxX: maximum width
  * @maxY: maximum height
- * @maxPitch: max pitch
  *
- * LOCKING:
- * Caller must hold a lock protecting @mode_list.
- *
- * The DRM device (@dev) has size and pitch limits.  Here we validate the
- * modes we probed for @dev against those limits and set their status as
- * necessary.
+ * This function is a helper which can be used to validate modes against size
+ * limitations of the DRM device/connector. If a mode is too big its status
+ * memeber is updated with the appropriate validation failure code. The list
+ * itself is not changed.
  */
 void drm_mode_validate_size(struct drm_device *dev,
                             struct list_head *mode_list,
-                            int maxX, int maxY, int maxPitch)
+                            int maxX, int maxY)
 {
         struct drm_display_mode *mode;
 
         list_for_each_entry(mode, mode_list, head) {
-                if (maxPitch > 0 && mode->hdisplay > maxPitch)
-                        mode->status = MODE_BAD_WIDTH;
-
                 if (maxX > 0 && mode->hdisplay > maxX)
                         mode->status = MODE_VIRTUAL_X;
 
@@ -934,12 +939,10 @@ EXPORT_SYMBOL(drm_mode_validate_size);
  * @mode_list: list of modes to check
  * @verbose: be verbose about it
  *
- * LOCKING:
- * Caller must hold a lock protecting @mode_list.
- *
- * Once mode list generation is complete, a caller can use this routine to
- * remove invalid modes from a mode list.  If any of the modes have a
- * status other than %MODE_OK, they are removed from @mode_list and freed.
+ * This helper function can be used to prune a display mode list after
+ * validation has been completed. All modes who's status is not MODE_OK will be
+ * removed from the list, and if @verbose the status code and mode name is also
+ * printed to dmesg.
  */
 void drm_mode_prune_invalid(struct drm_device *dev,
                             struct list_head *mode_list, bool verbose)
@@ -966,13 +969,10 @@ EXPORT_SYMBOL(drm_mode_prune_invalid);
  * @lh_a: list_head for first mode
  * @lh_b: list_head for second mode
  *
- * LOCKING:
- * None.
- *
  * Compare two modes, given by @lh_a and @lh_b, returning a value indicating
  * which is better.
  *
- * RETURNS:
+ * Returns:
  * Negative if @lh_a is better than @lh_b, zero if they're equivalent, or
  * positive if @lh_b is better than @lh_a.
  */
@@ -1000,12 +1000,9 @@ static int drm_mode_compare(void *priv, struct list_head *lh_a, struct list_head
 
 /**
  * drm_mode_sort - sort mode list
- * @mode_list: list to sort
+ * @mode_list: list of drm_display_mode structures to sort
  *
- * LOCKING:
- * Caller must hold a lock protecting @mode_list.
- *
- * Sort @mode_list by favorability, putting good modes first.
+ * Sort @mode_list by favorability, moving good modes to the head of the list.
  */
 void drm_mode_sort(struct list_head *mode_list)
 {
@@ -1017,13 +1014,12 @@ EXPORT_SYMBOL(drm_mode_sort);
  * drm_mode_connector_list_update - update the mode list for the connector
  * @connector: the connector to update
  *
- * LOCKING:
- * Caller must hold a lock protecting @mode_list.
- *
  * This moves the modes from the @connector probed_modes list
  * to the actual mode list. It compares the probed mode against the current
- * list and only adds different modes. All modes unverified after this point
- * will be removed by the prune invalid modes.
+ * list and only adds different/new modes.
+ *
+ * This is just a helper functions doesn't validate any modes itself and also
+ * doesn't prune any invalid modes. Callers need to do that themselves.
  */
 void drm_mode_connector_list_update(struct drm_connector *connector)
 {
@@ -1031,6 +1027,8 @@ void drm_mode_connector_list_update(struct drm_connector *connector)
         struct drm_display_mode *pmode, *pt;
         int found_it;
 
+        WARN_ON(!mutex_is_locked(&connector->dev->mode_config.mutex));
+
         list_for_each_entry_safe(pmode, pt, &connector->probed_modes,
                                  head) {
                 found_it = 0;
@@ -1056,17 +1054,25 @@ void drm_mode_connector_list_update(struct drm_connector *connector)
 EXPORT_SYMBOL(drm_mode_connector_list_update);
 
 /**
- * drm_mode_parse_command_line_for_connector - parse command line for connector
- * @mode_option - per connector mode option
- * @connector - connector to parse line for
+ * drm_mode_parse_command_line_for_connector - parse command line modeline for connector
+ * @mode_option: optional per connector mode option
+ * @connector: connector to parse modeline for
+ * @mode: preallocated drm_cmdline_mode structure to fill out
+ *
+ * This parses @mode_option command line modeline for modes and options to
+ * configure the connector. If @mode_option is NULL the default command line
+ * modeline in fb_mode_option will be parsed instead.
  *
- * This parses the connector specific then generic command lines for
- * modes and options to configure the connector.
+ * This uses the same parameters as the fb modedb.c, except for an extra
+ * force-enable, force-enable-digital and force-disable bit at the end:
  *
- * This uses the same parameters as the fb modedb.c, except for extra
  *      <xres>x<yres>[M][R][-<bpp>][@<refresh>][i][m][eDd]
  *
- * enable/enable Digital/disable bit at the end
+ * The intermediate drm_cmdline_mode structure is required to store additional
+ * options from the command line modline like the force-enabel/disable flag.
+ *
+ * Returns:
+ * True if a valid modeline has been parsed, false otherwise.
  */
 bool drm_mode_parse_command_line_for_connector(const char *mode_option,
                                                struct drm_connector *connector,
@@ -1219,6 +1225,14 @@ done:
 }
 EXPORT_SYMBOL(drm_mode_parse_command_line_for_connector);
 
+/**
+ * drm_mode_create_from_cmdline_mode - convert a command line modeline into a DRM display mode
+ * @dev: DRM device to create the new mode for
+ * @cmd: input command line modeline
+ *
+ * Returns:
+ * Pointer to converted mode on success, NULL on error.
+ */
 struct drm_display_mode *
 drm_mode_create_from_cmdline_mode(struct drm_device *dev,
                                   struct drm_cmdline_mode *cmd)
diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c
index f7af69bcf3f4..9c696a5ad74d 100644
--- a/drivers/gpu/drm/drm_pci.c
+++ b/drivers/gpu/drm/drm_pci.c
@@ -351,7 +351,7 @@ err_agp:
         drm_pci_agp_destroy(dev);
         pci_disable_device(pdev);
 err_free:
-        drm_dev_free(dev);
+        drm_dev_unref(dev);
         return ret;
 }
 EXPORT_SYMBOL(drm_get_pci_dev);
diff --git a/drivers/gpu/drm/drm_plane_helper.c b/drivers/gpu/drm/drm_plane_helper.c
new file mode 100644
index 000000000000..e768d35ff22e
--- /dev/null
+++ b/drivers/gpu/drm/drm_plane_helper.c
@@ -0,0 +1,333 @@
+/*
+ * Copyright (C) 2014 Intel Corporation
+ *
+ * DRM universal plane helper functions
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/list.h>
+#include <drm/drmP.h>
+#include <drm/drm_rect.h>
+
+#define SUBPIXEL_MASK 0xffff
+
+/*
+ * This is the minimal list of formats that seem to be safe for modeset use
+ * with all current DRM drivers.  Most hardware can actually support more
+ * formats than this and drivers may specify a more accurate list when
+ * creating the primary plane.  However drivers that still call
+ * drm_plane_init() will use this minimal format list as the default.
+ */
+const static uint32_t safe_modeset_formats[] = {
+       DRM_FORMAT_XRGB8888,
+       DRM_FORMAT_ARGB8888,
+};
+
+/*
+ * Returns the connectors currently associated with a CRTC.  This function
+ * should be called twice:  once with a NULL connector list to retrieve
+ * the list size, and once with the properly allocated list to be filled in.
+ */
+static int get_connectors_for_crtc(struct drm_crtc *crtc,
+                                   struct drm_connector **connector_list,
+                                   int num_connectors)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_connector *connector;
+        int count = 0;
+
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head)
+                if (connector->encoder && connector->encoder->crtc == crtc) {
+                        if (connector_list != NULL && count < num_connectors)
+                                *(connector_list++) = connector;
+
+                        count++;
+                }
+
+        return count;
+}
+
+/**
+ * drm_primary_helper_update() - Helper for primary plane update
+ * @plane: plane object to update
+ * @crtc: owning CRTC of owning plane
+ * @fb: framebuffer to flip onto plane
+ * @crtc_x: x offset of primary plane on crtc
+ * @crtc_y: y offset of primary plane on crtc
+ * @crtc_w: width of primary plane rectangle on crtc
+ * @crtc_h: height of primary plane rectangle on crtc
+ * @src_x: x offset of @fb for panning
+ * @src_y: y offset of @fb for panning
+ * @src_w: width of source rectangle in @fb
+ * @src_h: height of source rectangle in @fb
+ *
+ * Provides a default plane update handler for primary planes.  This is handler
+ * is called in response to a userspace SetPlane operation on the plane with a
+ * non-NULL framebuffer.  We call the driver's modeset handler to update the
+ * framebuffer.
+ *
+ * SetPlane() on a primary plane of a disabled CRTC is not supported, and will
+ * return an error.
+ *
+ * Note that we make some assumptions about hardware limitations that may not be
+ * true for all hardware --
+ *   1) Primary plane cannot be repositioned.
+ *   2) Primary plane cannot be scaled.
+ *   3) Primary plane must cover the entire CRTC.
+ *   4) Subpixel positioning is not supported.
+ * Drivers for hardware that don't have these restrictions can provide their
+ * own implementation rather than using this helper.
+ *
+ * RETURNS:
+ * Zero on success, error code on failure
+ */
+int drm_primary_helper_update(struct drm_plane *plane, struct drm_crtc *crtc,
+                              struct drm_framebuffer *fb,
+                              int crtc_x, int crtc_y,
+                              unsigned int crtc_w, unsigned int crtc_h,
+                              uint32_t src_x, uint32_t src_y,
+                              uint32_t src_w, uint32_t src_h)
+{
+        struct drm_mode_set set = {
+                .crtc = crtc,
+                .fb = fb,
+                .mode = &crtc->mode,
+                .x = src_x >> 16,
+                .y = src_y >> 16,
+        };
+        struct drm_rect dest = {
+                .x1 = crtc_x,
+                .y1 = crtc_y,
+                .x2 = crtc_x + crtc_w,
+                .y2 = crtc_y + crtc_h,
+        };
+        struct drm_rect clip = {
+                .x2 = crtc->mode.hdisplay,
+                .y2 = crtc->mode.vdisplay,
+        };
+        struct drm_connector **connector_list;
+        struct drm_framebuffer *tmpfb;
+        int num_connectors, ret;
+
+        if (!crtc->enabled) {
+                DRM_DEBUG_KMS("Cannot update primary plane of a disabled CRTC.\n");
+                return -EINVAL;
+        }
+
+        /* Disallow subpixel positioning */
+        if ((src_x | src_y | src_w | src_h) & SUBPIXEL_MASK) {
+                DRM_DEBUG_KMS("Primary plane does not support subpixel positioning\n");
+                return -EINVAL;
+        }
+
+        /* Primary planes are locked to their owning CRTC */
+        if (plane->possible_crtcs != drm_crtc_mask(crtc)) {
+                DRM_DEBUG_KMS("Cannot change primary plane CRTC\n");
+                return -EINVAL;
+        }
+
+        /* Disallow scaling */
+        if (crtc_w != src_w || crtc_h != src_h) {
+                DRM_DEBUG_KMS("Can't scale primary plane\n");
+                return -EINVAL;
+        }
+
+        /* Make sure primary plane covers entire CRTC */
+        drm_rect_intersect(&dest, &clip);
+        if (dest.x1 != 0 || dest.y1 != 0 ||
+            dest.x2 != crtc->mode.hdisplay || dest.y2 != crtc->mode.vdisplay) {
+                DRM_DEBUG_KMS("Primary plane must cover entire CRTC\n");
+                return -EINVAL;
+        }
+
+        /* Framebuffer must be big enough to cover entire plane */
+        ret = drm_crtc_check_viewport(crtc, crtc_x, crtc_y, &crtc->mode, fb);
+        if (ret)
+                return ret;
+
+        /* Find current connectors for CRTC */
+        num_connectors = get_connectors_for_crtc(crtc, NULL, 0);
+        BUG_ON(num_connectors == 0);
+        connector_list = kzalloc(num_connectors * sizeof(*connector_list),
+                                 GFP_KERNEL);
+        if (!connector_list)
+                return -ENOMEM;
+        get_connectors_for_crtc(crtc, connector_list, num_connectors);
+
+        set.connectors = connector_list;
+        set.num_connectors = num_connectors;
+
+        /*
+         * set_config() adjusts crtc->primary->fb; however the DRM setplane
+         * code that called us expects to handle the framebuffer update and
+         * reference counting; save and restore the current fb before
+         * calling it.
+         *
+         * N.B., we call set_config() directly here rather than using
+         * drm_mode_set_config_internal.  We're reprogramming the same
+         * connectors that were already in use, so we shouldn't need the extra
+         * cross-CRTC fb refcounting to accomodate stealing connectors.
+         * drm_mode_setplane() already handles the basic refcounting for the
+         * framebuffers involved in this operation.
+         */
+        tmpfb = plane->fb;
+        ret = crtc->funcs->set_config(&set);
+        plane->fb = tmpfb;
+
+        kfree(connector_list);
+        return ret;
+}
+EXPORT_SYMBOL(drm_primary_helper_update);
+
+/**
+ * drm_primary_helper_disable() - Helper for primary plane disable
+ * @plane: plane to disable
+ *
+ * Provides a default plane disable handler for primary planes.  This is handler
+ * is called in response to a userspace SetPlane operation on the plane with a
+ * NULL framebuffer parameter.  We call the driver's modeset handler with a NULL
+ * framebuffer to disable the CRTC if no other planes are currently enabled.
+ * If other planes are still enabled on the same CRTC, we return -EBUSY.
+ *
+ * Note that some hardware may be able to disable the primary plane without
+ * disabling the whole CRTC.  Drivers for such hardware should provide their
+ * own disable handler that disables just the primary plane (and they'll likely
+ * need to provide their own update handler as well to properly re-enable a
+ * disabled primary plane).
+ *
+ * RETURNS:
+ * Zero on success, error code on failure
+ */
+int drm_primary_helper_disable(struct drm_plane *plane)
+{
+        struct drm_plane *p;
+        struct drm_mode_set set = {
+                .crtc = plane->crtc,
+                .fb = NULL,
+        };
+
+        if (plane->crtc == NULL || plane->fb == NULL)
+                /* Already disabled */
+                return 0;
+
+        list_for_each_entry(p, &plane->dev->mode_config.plane_list, head)
+                if (p != plane && p->fb) {
+                        DRM_DEBUG_KMS("Cannot disable primary plane while other planes are still active on CRTC.\n");
+                        return -EBUSY;
+                }
+
+        /*
+         * N.B.  We call set_config() directly here rather than
+         * drm_mode_set_config_internal() since drm_mode_setplane() already
+         * handles the basic refcounting and we don't need the special
+         * cross-CRTC refcounting (no chance of stealing connectors from
+         * other CRTC's with this update).
+         */
+        return plane->crtc->funcs->set_config(&set);
+}
+EXPORT_SYMBOL(drm_primary_helper_disable);
+
+/**
+ * drm_primary_helper_destroy() - Helper for primary plane destruction
+ * @plane: plane to destroy
+ *
+ * Provides a default plane destroy handler for primary planes.  This handler
+ * is called during CRTC destruction.  We disable the primary plane, remove
+ * it from the DRM plane list, and deallocate the plane structure.
+ */
+void drm_primary_helper_destroy(struct drm_plane *plane)
+{
+        plane->funcs->disable_plane(plane);
+        drm_plane_cleanup(plane);
+        kfree(plane);
+}
+EXPORT_SYMBOL(drm_primary_helper_destroy);
+
+const struct drm_plane_funcs drm_primary_helper_funcs = {
+        .update_plane = drm_primary_helper_update,
+        .disable_plane = drm_primary_helper_disable,
+        .destroy = drm_primary_helper_destroy,
+};
+EXPORT_SYMBOL(drm_primary_helper_funcs);
+
+/**
+ * drm_primary_helper_create_plane() - Create a generic primary plane
+ * @dev: drm device
+ * @formats: pixel formats supported, or NULL for a default safe list
+ * @num_formats: size of @formats; ignored if @formats is NULL
+ *
+ * Allocates and initializes a primary plane that can be used with the primary
+ * plane helpers.  Drivers that wish to use driver-specific plane structures or
+ * provide custom handler functions may perform their own allocation and
+ * initialization rather than calling this function.
+ */
+struct drm_plane *drm_primary_helper_create_plane(struct drm_device *dev,
+                                                  const uint32_t *formats,
+                                                  int num_formats)
+{
+        struct drm_plane *primary;
+        int ret;
+
+        primary = kzalloc(sizeof(*primary), GFP_KERNEL);
+        if (primary == NULL) {
+                DRM_DEBUG_KMS("Failed to allocate primary plane\n");
+                return NULL;
+        }
+
+        if (formats == NULL) {
+                formats = safe_modeset_formats;
+                num_formats = ARRAY_SIZE(safe_modeset_formats);
+        }
+
+        /* possible_crtc's will be filled in later by crtc_init */
+        ret = drm_plane_init(dev, primary, 0, &drm_primary_helper_funcs,
+                             formats, num_formats,
+                             DRM_PLANE_TYPE_PRIMARY);
+        if (ret) {
+                kfree(primary);
+                primary = NULL;
+        }
+
+        return primary;
+}
+EXPORT_SYMBOL(drm_primary_helper_create_plane);
+
+/**
+ * drm_crtc_init - Legacy CRTC initialization function
+ * @dev: DRM device
+ * @crtc: CRTC object to init
+ * @funcs: callbacks for the new CRTC
+ *
+ * Initialize a CRTC object with a default helper-provided primary plane and no
+ * cursor plane.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
+                  const struct drm_crtc_funcs *funcs)
+{
+        struct drm_plane *primary;
+
+        primary = drm_primary_helper_create_plane(dev, NULL, 0);
+        return drm_crtc_init_with_planes(dev, crtc, primary, NULL, funcs);
+}
+EXPORT_SYMBOL(drm_crtc_init);
diff --git a/drivers/gpu/drm/drm_platform.c b/drivers/gpu/drm/drm_platform.c
index 21fc82006b78..319ff5385601 100644
--- a/drivers/gpu/drm/drm_platform.c
+++ b/drivers/gpu/drm/drm_platform.c
@@ -64,7 +64,7 @@ static int drm_get_platform_dev(struct platform_device *platdev,
         return 0;
 
 err_free:
-        drm_dev_free(dev);
+        drm_dev_unref(dev);
         return ret;
 }
 
diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c
index bb516fdd195d..304ca8cacbc4 100644
--- a/drivers/gpu/drm/drm_prime.c
+++ b/drivers/gpu/drm/drm_prime.c
@@ -68,7 +68,8 @@ struct drm_prime_attachment {
         enum dma_data_direction dir;
 };
 
-static int drm_prime_add_buf_handle(struct drm_prime_file_private *prime_fpriv, struct dma_buf *dma_buf, uint32_t handle)
+static int drm_prime_add_buf_handle(struct drm_prime_file_private *prime_fpriv,
+                                    struct dma_buf *dma_buf, uint32_t handle)
 {
         struct drm_prime_member *member;
 
@@ -174,7 +175,7 @@ void drm_prime_remove_buf_handle_locked(struct drm_prime_file_private *prime_fpr
 }
 
 static struct sg_table *drm_gem_map_dma_buf(struct dma_buf_attachment *attach,
-                enum dma_data_direction dir)
+                                            enum dma_data_direction dir)
 {
         struct drm_prime_attachment *prime_attach = attach->priv;
         struct drm_gem_object *obj = attach->dmabuf->priv;
@@ -211,11 +212,19 @@ static struct sg_table *drm_gem_map_dma_buf(struct dma_buf_attachment *attach,
 }
 
 static void drm_gem_unmap_dma_buf(struct dma_buf_attachment *attach,
-                struct sg_table *sgt, enum dma_data_direction dir)
+                                  struct sg_table *sgt,
+                                  enum dma_data_direction dir)
 {
         /* nothing to be done here */
 }
 
+/**
+ * drm_gem_dmabuf_release - dma_buf release implementation for GEM
+ * @dma_buf: buffer to be released
+ *
+ * Generic release function for dma_bufs exported as PRIME buffers. GEM drivers
+ * must use this in their dma_buf ops structure as the release callback.
+ */
 void drm_gem_dmabuf_release(struct dma_buf *dma_buf)
 {
         struct drm_gem_object *obj = dma_buf->priv;
@@ -242,30 +251,30 @@ static void drm_gem_dmabuf_vunmap(struct dma_buf *dma_buf, void *vaddr)
 }
 
 static void *drm_gem_dmabuf_kmap_atomic(struct dma_buf *dma_buf,
-                unsigned long page_num)
+                                        unsigned long page_num)
 {
         return NULL;
 }
 
 static void drm_gem_dmabuf_kunmap_atomic(struct dma_buf *dma_buf,
-                unsigned long page_num, void *addr)
+                                         unsigned long page_num, void *addr)
 {
 
 }
 static void *drm_gem_dmabuf_kmap(struct dma_buf *dma_buf,
-                unsigned long page_num)
+                                 unsigned long page_num)
 {
         return NULL;
 }
 
 static void drm_gem_dmabuf_kunmap(struct dma_buf *dma_buf,
-                unsigned long page_num, void *addr)
+                                  unsigned long page_num, void *addr)
 {
 
 }
 
 static int drm_gem_dmabuf_mmap(struct dma_buf *dma_buf,
-                struct vm_area_struct *vma)
+                               struct vm_area_struct *vma)
 {
         struct drm_gem_object *obj = dma_buf->priv;
         struct drm_device *dev = obj->dev;
@@ -315,6 +324,15 @@ static const struct dma_buf_ops drm_gem_prime_dmabuf_ops =  {
  *    driver's scatter/gather table
  */
 
+/**
+ * drm_gem_prime_export - helper library implemention of the export callback
+ * @dev: drm_device to export from
+ * @obj: GEM object to export
+ * @flags: flags like DRM_CLOEXEC
+ *
+ * This is the implementation of the gem_prime_export functions for GEM drivers
+ * using the PRIME helpers.
+ */
 struct dma_buf *drm_gem_prime_export(struct drm_device *dev,
                                      struct drm_gem_object *obj, int flags)
 {
@@ -355,9 +373,23 @@ static struct dma_buf *export_and_register_object(struct drm_device *dev,
         return dmabuf;
 }
 
+/**
+ * drm_gem_prime_handle_to_fd - PRIME export function for GEM drivers
+ * @dev: dev to export the buffer from
+ * @file_priv: drm file-private structure
+ * @handle: buffer handle to export
+ * @flags: flags like DRM_CLOEXEC
+ * @prime_fd: pointer to storage for the fd id of the create dma-buf
+ *
+ * This is the PRIME export function which must be used mandatorily by GEM
+ * drivers to ensure correct lifetime management of the underlying GEM object.
+ * The actual exporting from GEM object to a dma-buf is done through the
+ * gem_prime_export driver callback.
+ */
 int drm_gem_prime_handle_to_fd(struct drm_device *dev,
-                struct drm_file *file_priv, uint32_t handle, uint32_t flags,
-                int *prime_fd)
+                               struct drm_file *file_priv, uint32_t handle,
+                               uint32_t flags,
+                               int *prime_fd)
 {
         struct drm_gem_object *obj;
         int ret = 0;
@@ -441,6 +473,14 @@ out_unlock:
 }
 EXPORT_SYMBOL(drm_gem_prime_handle_to_fd);
 
+/**
+ * drm_gem_prime_import - helper library implemention of the import callback
+ * @dev: drm_device to import into
+ * @dma_buf: dma-buf object to import
+ *
+ * This is the implementation of the gem_prime_import functions for GEM drivers
+ * using the PRIME helpers.
+ */
 struct drm_gem_object *drm_gem_prime_import(struct drm_device *dev,
                                             struct dma_buf *dma_buf)
 {
@@ -496,8 +536,21 @@ fail_detach:
 }
 EXPORT_SYMBOL(drm_gem_prime_import);
 
+/**
+ * drm_gem_prime_fd_to_handle - PRIME import function for GEM drivers
+ * @dev: dev to export the buffer from
+ * @file_priv: drm file-private structure
+ * @prime_fd: fd id of the dma-buf which should be imported
+ * @handle: pointer to storage for the handle of the imported buffer object
+ *
+ * This is the PRIME import function which must be used mandatorily by GEM
+ * drivers to ensure correct lifetime management of the underlying GEM object.
+ * The actual importing of GEM object from the dma-buf is done through the
+ * gem_import_export driver callback.
+ */
 int drm_gem_prime_fd_to_handle(struct drm_device *dev,
-                struct drm_file *file_priv, int prime_fd, uint32_t *handle)
+                               struct drm_file *file_priv, int prime_fd,
+                               uint32_t *handle)
 {
         struct dma_buf *dma_buf;
         struct drm_gem_object *obj;
@@ -598,12 +651,14 @@ int drm_prime_fd_to_handle_ioctl(struct drm_device *dev, void *data,
                         args->fd, &args->handle);
 }
 
-/*
- * drm_prime_pages_to_sg
+/**
+ * drm_prime_pages_to_sg - converts a page array into an sg list
+ * @pages: pointer to the array of page pointers to convert
+ * @nr_pages: length of the page vector
  *
- * this helper creates an sg table object from a set of pages
+ * This helper creates an sg table object from a set of pages
  * the driver is responsible for mapping the pages into the
- * importers address space
+ * importers address space for use with dma_buf itself.
  */
 struct sg_table *drm_prime_pages_to_sg(struct page **pages, int nr_pages)
 {
@@ -628,9 +683,16 @@ out:
 }
 EXPORT_SYMBOL(drm_prime_pages_to_sg);
 
-/* export an sg table into an array of pages and addresses
-   this is currently required by the TTM driver in order to do correct fault
-   handling */
+/**
+ * drm_prime_sg_to_page_addr_arrays - convert an sg table into a page array
+ * @sgt: scatter-gather table to convert
+ * @pages: array of page pointers to store the page array in
+ * @addrs: optional array to store the dma bus address of each page
+ * @max_pages: size of both the passed-in arrays
+ *
+ * Exports an sg table into an array of pages and addresses. This is currently
+ * required by the TTM driver in order to do correct fault handling.
+ */
 int drm_prime_sg_to_page_addr_arrays(struct sg_table *sgt, struct page **pages,
                                      dma_addr_t *addrs, int max_pages)
 {
@@ -663,7 +725,15 @@ int drm_prime_sg_to_page_addr_arrays(struct sg_table *sgt, struct page **pages,
         return 0;
 }
 EXPORT_SYMBOL(drm_prime_sg_to_page_addr_arrays);
-/* helper function to cleanup a GEM/prime object */
+
+/**
+ * drm_prime_gem_destroy - helper to clean up a PRIME-imported GEM object
+ * @obj: GEM object which was created from a dma-buf
+ * @sg: the sg-table which was pinned at import time
+ *
+ * This is the cleanup functions which GEM drivers need to call when they use
+ * @drm_gem_prime_import to import dma-bufs.
+ */
 void drm_prime_gem_destroy(struct drm_gem_object *obj, struct sg_table *sg)
 {
         struct dma_buf_attachment *attach;
@@ -683,11 +753,9 @@ void drm_prime_init_file_private(struct drm_prime_file_private *prime_fpriv)
         INIT_LIST_HEAD(&prime_fpriv->head);
         mutex_init(&prime_fpriv->lock);
 }
-EXPORT_SYMBOL(drm_prime_init_file_private);
 
 void drm_prime_destroy_file_private(struct drm_prime_file_private *prime_fpriv)
 {
         /* by now drm_gem_release should've made sure the list is empty */
         WARN_ON(!list_empty(&prime_fpriv->head));
 }
-EXPORT_SYMBOL(drm_prime_destroy_file_private);
diff --git a/drivers/gpu/drm/drm_stub.c b/drivers/gpu/drm/drm_stub.c
index 98a33c580ca1..4c24c3ac1efa 100644
--- a/drivers/gpu/drm/drm_stub.c
+++ b/drivers/gpu/drm/drm_stub.c
@@ -31,8 +31,10 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
+#include <linux/fs.h>
 #include <linux/module.h>
 #include <linux/moduleparam.h>
+#include <linux/mount.h>
 #include <linux/slab.h>
 #include <drm/drmP.h>
 #include <drm/drm_core.h>
@@ -43,6 +45,10 @@ EXPORT_SYMBOL(drm_debug);
 unsigned int drm_rnodes = 0;    /* 1 to enable experimental render nodes API */
 EXPORT_SYMBOL(drm_rnodes);
 
+/* 1 to allow user space to request universal planes (experimental) */
+unsigned int drm_universal_planes = 0;
+EXPORT_SYMBOL(drm_universal_planes);
+
 unsigned int drm_vblank_offdelay = 5000;    /* Default to 5000 msecs. */
 EXPORT_SYMBOL(drm_vblank_offdelay);
 
@@ -66,10 +72,12 @@ MODULE_PARM_DESC(timestamp_monotonic, "Use monotonic timestamps");
 
 module_param_named(debug, drm_debug, int, 0600);
 module_param_named(rnodes, drm_rnodes, int, 0600);
+module_param_named(universal_planes, drm_universal_planes, int, 0600);
 module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
 module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
 module_param_named(timestamp_monotonic, drm_timestamp_monotonic, int, 0600);
 
+static DEFINE_SPINLOCK(drm_minor_lock);
 struct idr drm_minors_idr;
 
 struct class *drm_class;
@@ -94,48 +102,20 @@ int drm_err(const char *func, const char *format, ...)
 }
 EXPORT_SYMBOL(drm_err);
 
-void drm_ut_debug_printk(unsigned int request_level,
-                         const char *prefix,
-                         const char *function_name,
-                         const char *format, ...)
+void drm_ut_debug_printk(const char *function_name, const char *format, ...)
 {
         struct va_format vaf;
         va_list args;
 
-        if (drm_debug & request_level) {
-                va_start(args, format);
-                vaf.fmt = format;
-                vaf.va = &args;
-
-                if (function_name)
-                        printk(KERN_DEBUG "[%s:%s], %pV", prefix,
-                               function_name, &vaf);
-                else
-                        printk(KERN_DEBUG "%pV", &vaf);
-                va_end(args);
-        }
-}
-EXPORT_SYMBOL(drm_ut_debug_printk);
-
-static int drm_minor_get_id(struct drm_device *dev, int type)
-{
-        int ret;
-        int base = 0, limit = 63;
-
-        if (type == DRM_MINOR_CONTROL) {
-                base += 64;
-                limit = base + 63;
-        } else if (type == DRM_MINOR_RENDER) {
-                base += 128;
-                limit = base + 63;
-        }
+        va_start(args, format);
+        vaf.fmt = format;
+        vaf.va = &args;
 
-        mutex_lock(&dev->struct_mutex);
-        ret = idr_alloc(&drm_minors_idr, NULL, base, limit, GFP_KERNEL);
-        mutex_unlock(&dev->struct_mutex);
+        printk(KERN_DEBUG "[" DRM_NAME ":%s] %pV", function_name, &vaf);
 
-        return ret == -ENOSPC ? -EINVAL : ret;
+        va_end(args);
 }
+EXPORT_SYMBOL(drm_ut_debug_printk);
 
 struct drm_master *drm_master_create(struct drm_minor *minor)
 {
@@ -152,8 +132,6 @@ struct drm_master *drm_master_create(struct drm_minor *minor)
         INIT_LIST_HEAD(&master->magicfree);
         master->minor = minor;
 
-        list_add_tail(&master->head, &minor->master_list);
-
         return master;
 }
 
@@ -171,8 +149,7 @@ static void drm_master_destroy(struct kref *kref)
         struct drm_device *dev = master->minor->dev;
         struct drm_map_list *r_list, *list_temp;
 
-        list_del(&master->head);
-
+        mutex_lock(&dev->struct_mutex);
         if (dev->driver->master_destroy)
                 dev->driver->master_destroy(dev, master);
 
@@ -200,6 +177,7 @@ static void drm_master_destroy(struct kref *kref)
 
         drm_ht_remove(&master->magiclist);
 
+        mutex_unlock(&dev->struct_mutex);
         kfree(master);
 }
 
@@ -215,19 +193,20 @@ int drm_setmaster_ioctl(struct drm_device *dev, void *data,
 {
         int ret = 0;
 
+        mutex_lock(&dev->master_mutex);
         if (file_priv->is_master)
-                return 0;
-
-        if (file_priv->minor->master && file_priv->minor->master != file_priv->master)
-                return -EINVAL;
+                goto out_unlock;
 
-        if (!file_priv->master)
-                return -EINVAL;
+        if (file_priv->minor->master) {
+                ret = -EINVAL;
+                goto out_unlock;
+        }
 
-        if (file_priv->minor->master)
-                return -EINVAL;
+        if (!file_priv->master) {
+                ret = -EINVAL;
+                goto out_unlock;
+        }
 
-        mutex_lock(&dev->struct_mutex);
         file_priv->minor->master = drm_master_get(file_priv->master);
         file_priv->is_master = 1;
         if (dev->driver->master_set) {
@@ -237,142 +216,211 @@ int drm_setmaster_ioctl(struct drm_device *dev, void *data,
                         drm_master_put(&file_priv->minor->master);
                 }
         }
-        mutex_unlock(&dev->struct_mutex);
 
+out_unlock:
+        mutex_unlock(&dev->master_mutex);
         return ret;
 }
 
 int drm_dropmaster_ioctl(struct drm_device *dev, void *data,
                          struct drm_file *file_priv)
 {
+        int ret = -EINVAL;
+
+        mutex_lock(&dev->master_mutex);
         if (!file_priv->is_master)
-                return -EINVAL;
+                goto out_unlock;
 
         if (!file_priv->minor->master)
-                return -EINVAL;
+                goto out_unlock;
 
-        mutex_lock(&dev->struct_mutex);
+        ret = 0;
         if (dev->driver->master_drop)
                 dev->driver->master_drop(dev, file_priv, false);
         drm_master_put(&file_priv->minor->master);
         file_priv->is_master = 0;
-        mutex_unlock(&dev->struct_mutex);
-        return 0;
+
+out_unlock:
+        mutex_unlock(&dev->master_mutex);
+        return ret;
 }
 
-/**
- * drm_get_minor - Allocate and register new DRM minor
- * @dev: DRM device
- * @minor: Pointer to where new minor is stored
- * @type: Type of minor
- *
- * Allocate a new minor of the given type and register it. A pointer to the new
- * minor is returned in @minor.
- * Caller must hold the global DRM mutex.
+/*
+ * DRM Minors
+ * A DRM device can provide several char-dev interfaces on the DRM-Major. Each
+ * of them is represented by a drm_minor object. Depending on the capabilities
+ * of the device-driver, different interfaces are registered.
  *
- * RETURNS:
- * 0 on success, negative error code on failure.
+ * Minors can be accessed via dev->$minor_name. This pointer is either
+ * NULL or a valid drm_minor pointer and stays valid as long as the device is
+ * valid. This means, DRM minors have the same life-time as the underlying
+ * device. However, this doesn't mean that the minor is active. Minors are
+ * registered and unregistered dynamically according to device-state.
  */
-static int drm_get_minor(struct drm_device *dev, struct drm_minor **minor,
-                         int type)
+
+static struct drm_minor **drm_minor_get_slot(struct drm_device *dev,
+                                             unsigned int type)
+{
+        switch (type) {
+        case DRM_MINOR_LEGACY:
+                return &dev->primary;
+        case DRM_MINOR_RENDER:
+                return &dev->render;
+        case DRM_MINOR_CONTROL:
+                return &dev->control;
+        default:
+                return NULL;
+        }
+}
+
+static int drm_minor_alloc(struct drm_device *dev, unsigned int type)
+{
+        struct drm_minor *minor;
+
+        minor = kzalloc(sizeof(*minor), GFP_KERNEL);
+        if (!minor)
+                return -ENOMEM;
+
+        minor->type = type;
+        minor->dev = dev;
+
+        *drm_minor_get_slot(dev, type) = minor;
+        return 0;
+}
+
+static void drm_minor_free(struct drm_device *dev, unsigned int type)
+{
+        struct drm_minor **slot;
+
+        slot = drm_minor_get_slot(dev, type);
+        if (*slot) {
+                kfree(*slot);
+                *slot = NULL;
+        }
+}
+
+static int drm_minor_register(struct drm_device *dev, unsigned int type)
 {
         struct drm_minor *new_minor;
+        unsigned long flags;
         int ret;
         int minor_id;
 
         DRM_DEBUG("\n");
 
-        minor_id = drm_minor_get_id(dev, type);
+        new_minor = *drm_minor_get_slot(dev, type);
+        if (!new_minor)
+                return 0;
+
+        idr_preload(GFP_KERNEL);
+        spin_lock_irqsave(&drm_minor_lock, flags);
+        minor_id = idr_alloc(&drm_minors_idr,
+                             NULL,
+                             64 * type,
+                             64 * (type + 1),
+                             GFP_NOWAIT);
+        spin_unlock_irqrestore(&drm_minor_lock, flags);
+        idr_preload_end();
+
         if (minor_id < 0)
                 return minor_id;
 
-        new_minor = kzalloc(sizeof(struct drm_minor), GFP_KERNEL);
-        if (!new_minor) {
-                ret = -ENOMEM;
-                goto err_idr;
-        }
-
-        new_minor->type = type;
-        new_minor->device = MKDEV(DRM_MAJOR, minor_id);
-        new_minor->dev = dev;
         new_minor->index = minor_id;
-        INIT_LIST_HEAD(&new_minor->master_list);
-
-        idr_replace(&drm_minors_idr, new_minor, minor_id);
 
-#if defined(CONFIG_DEBUG_FS)
         ret = drm_debugfs_init(new_minor, minor_id, drm_debugfs_root);
         if (ret) {
                 DRM_ERROR("DRM: Failed to initialize /sys/kernel/debug/dri.\n");
-                goto err_mem;
+                goto err_id;
         }
-#endif
 
         ret = drm_sysfs_device_add(new_minor);
         if (ret) {
-                printk(KERN_ERR
-                       "DRM: Error sysfs_device_add.\n");
+                DRM_ERROR("DRM: Error sysfs_device_add.\n");
                 goto err_debugfs;
         }
-        *minor = new_minor;
+
+        /* replace NULL with @minor so lookups will succeed from now on */
+        spin_lock_irqsave(&drm_minor_lock, flags);
+        idr_replace(&drm_minors_idr, new_minor, new_minor->index);
+        spin_unlock_irqrestore(&drm_minor_lock, flags);
 
         DRM_DEBUG("new minor assigned %d\n", minor_id);
         return 0;
 
-
 err_debugfs:
-#if defined(CONFIG_DEBUG_FS)
         drm_debugfs_cleanup(new_minor);
-err_mem:
-#endif
-        kfree(new_minor);
-err_idr:
+err_id:
+        spin_lock_irqsave(&drm_minor_lock, flags);
         idr_remove(&drm_minors_idr, minor_id);
-        *minor = NULL;
+        spin_unlock_irqrestore(&drm_minor_lock, flags);
+        new_minor->index = 0;
         return ret;
 }
 
-/**
- * drm_unplug_minor - Unplug DRM minor
- * @minor: Minor to unplug
- *
- * Unplugs the given DRM minor but keeps the object. So after this returns,
- * minor->dev is still valid so existing open-files can still access it to get
- * device information from their drm_file ojects.
- * If the minor is already unplugged or if @minor is NULL, nothing is done.
- * The global DRM mutex must be held by the caller.
- */
-static void drm_unplug_minor(struct drm_minor *minor)
+static void drm_minor_unregister(struct drm_device *dev, unsigned int type)
 {
+        struct drm_minor *minor;
+        unsigned long flags;
+
+        minor = *drm_minor_get_slot(dev, type);
         if (!minor || !minor->kdev)
                 return;
 
-#if defined(CONFIG_DEBUG_FS)
-        drm_debugfs_cleanup(minor);
-#endif
+        spin_lock_irqsave(&drm_minor_lock, flags);
+        idr_remove(&drm_minors_idr, minor->index);
+        spin_unlock_irqrestore(&drm_minor_lock, flags);
+        minor->index = 0;
 
+        drm_debugfs_cleanup(minor);
         drm_sysfs_device_remove(minor);
-        idr_remove(&drm_minors_idr, minor->index);
 }
 
 /**
- * drm_put_minor - Destroy DRM minor
- * @minor: Minor to destroy
+ * drm_minor_acquire - Acquire a DRM minor
+ * @minor_id: Minor ID of the DRM-minor
+ *
+ * Looks up the given minor-ID and returns the respective DRM-minor object. The
+ * refence-count of the underlying device is increased so you must release this
+ * object with drm_minor_release().
  *
- * This calls drm_unplug_minor() on the given minor and then frees it. Nothing
- * is done if @minor is NULL. It is fine to call this on already unplugged
- * minors.
- * The global DRM mutex must be held by the caller.
+ * As long as you hold this minor, it is guaranteed that the object and the
+ * minor->dev pointer will stay valid! However, the device may get unplugged and
+ * unregistered while you hold the minor.
+ *
+ * Returns:
+ * Pointer to minor-object with increased device-refcount, or PTR_ERR on
+ * failure.
  */
-static void drm_put_minor(struct drm_minor *minor)
+struct drm_minor *drm_minor_acquire(unsigned int minor_id)
 {
-        if (!minor)
-                return;
+        struct drm_minor *minor;
+        unsigned long flags;
+
+        spin_lock_irqsave(&drm_minor_lock, flags);
+        minor = idr_find(&drm_minors_idr, minor_id);
+        if (minor)
+                drm_dev_ref(minor->dev);
+        spin_unlock_irqrestore(&drm_minor_lock, flags);
+
+        if (!minor) {
+                return ERR_PTR(-ENODEV);
+        } else if (drm_device_is_unplugged(minor->dev)) {
+                drm_dev_unref(minor->dev);
+                return ERR_PTR(-ENODEV);
+        }
 
-        DRM_DEBUG("release secondary minor %d\n", minor->index);
+        return minor;
+}
 
-        drm_unplug_minor(minor);
-        kfree(minor);
+/**
+ * drm_minor_release - Release DRM minor
+ * @minor: Pointer to DRM minor object
+ *
+ * Release a minor that was previously acquired via drm_minor_acquire().
+ */
+void drm_minor_release(struct drm_minor *minor)
+{
+        drm_dev_unref(minor->dev);
 }
 
 /**
@@ -392,18 +440,16 @@ void drm_put_dev(struct drm_device *dev)
         }
 
         drm_dev_unregister(dev);
-        drm_dev_free(dev);
+        drm_dev_unref(dev);
 }
 EXPORT_SYMBOL(drm_put_dev);
 
 void drm_unplug_dev(struct drm_device *dev)
 {
         /* for a USB device */
-        if (drm_core_check_feature(dev, DRIVER_MODESET))
-                drm_unplug_minor(dev->control);
-        if (dev->render)
-                drm_unplug_minor(dev->render);
-        drm_unplug_minor(dev->primary);
+        drm_minor_unregister(dev, DRM_MINOR_LEGACY);
+        drm_minor_unregister(dev, DRM_MINOR_RENDER);
+        drm_minor_unregister(dev, DRM_MINOR_CONTROL);
 
         mutex_lock(&drm_global_mutex);
 
@@ -416,6 +462,78 @@ void drm_unplug_dev(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_unplug_dev);
 
+/*
+ * DRM internal mount
+ * We want to be able to allocate our own "struct address_space" to control
+ * memory-mappings in VRAM (or stolen RAM, ...). However, core MM does not allow
+ * stand-alone address_space objects, so we need an underlying inode. As there
+ * is no way to allocate an independent inode easily, we need a fake internal
+ * VFS mount-point.
+ *
+ * The drm_fs_inode_new() function allocates a new inode, drm_fs_inode_free()
+ * frees it again. You are allowed to use iget() and iput() to get references to
+ * the inode. But each drm_fs_inode_new() call must be paired with exactly one
+ * drm_fs_inode_free() call (which does not have to be the last iput()).
+ * We use drm_fs_inode_*() to manage our internal VFS mount-point and share it
+ * between multiple inode-users. You could, technically, call
+ * iget() + drm_fs_inode_free() directly after alloc and sometime later do an
+ * iput(), but this way you'd end up with a new vfsmount for each inode.
+ */
+
+static int drm_fs_cnt;
+static struct vfsmount *drm_fs_mnt;
+
+static const struct dentry_operations drm_fs_dops = {
+        .d_dname        = simple_dname,
+};
+
+static const struct super_operations drm_fs_sops = {
+        .statfs         = simple_statfs,
+};
+
+static struct dentry *drm_fs_mount(struct file_system_type *fs_type, int flags,
+                                   const char *dev_name, void *data)
+{
+        return mount_pseudo(fs_type,
+                            "drm:",
+                            &drm_fs_sops,
+                            &drm_fs_dops,
+                            0x010203ff);
+}
+
+static struct file_system_type drm_fs_type = {
+        .name           = "drm",
+        .owner          = THIS_MODULE,
+        .mount          = drm_fs_mount,
+        .kill_sb        = kill_anon_super,
+};
+
+static struct inode *drm_fs_inode_new(void)
+{
+        struct inode *inode;
+        int r;
+
+        r = simple_pin_fs(&drm_fs_type, &drm_fs_mnt, &drm_fs_cnt);
+        if (r < 0) {
+                DRM_ERROR("Cannot mount pseudo fs: %d\n", r);
+                return ERR_PTR(r);
+        }
+
+        inode = alloc_anon_inode(drm_fs_mnt->mnt_sb);
+        if (IS_ERR(inode))
+                simple_release_fs(&drm_fs_mnt, &drm_fs_cnt);
+
+        return inode;
+}
+
+static void drm_fs_inode_free(struct inode *inode)
+{
+        if (inode) {
+                iput(inode);
+                simple_release_fs(&drm_fs_mnt, &drm_fs_cnt);
+        }
+}
+
 /**
  * drm_dev_alloc - Allocate new drm device
  * @driver: DRM driver to allocate device for
@@ -425,6 +543,9 @@ EXPORT_SYMBOL(drm_unplug_dev);
  * Call drm_dev_register() to advertice the device to user space and register it
  * with other core subsystems.
  *
+ * The initial ref-count of the object is 1. Use drm_dev_ref() and
+ * drm_dev_unref() to take and drop further ref-counts.
+ *
  * RETURNS:
  * Pointer to new DRM device, or NULL if out of memory.
  */
@@ -438,6 +559,7 @@ struct drm_device *drm_dev_alloc(struct drm_driver *driver,
         if (!dev)
                 return NULL;
 
+        kref_init(&dev->ref);
         dev->dev = parent;
         dev->driver = driver;
 
@@ -451,9 +573,33 @@ struct drm_device *drm_dev_alloc(struct drm_driver *driver,
         spin_lock_init(&dev->event_lock);
         mutex_init(&dev->struct_mutex);
         mutex_init(&dev->ctxlist_mutex);
+        mutex_init(&dev->master_mutex);
 
-        if (drm_ht_create(&dev->map_hash, 12))
+        dev->anon_inode = drm_fs_inode_new();
+        if (IS_ERR(dev->anon_inode)) {
+                ret = PTR_ERR(dev->anon_inode);
+                DRM_ERROR("Cannot allocate anonymous inode: %d\n", ret);
                 goto err_free;
+        }
+
+        if (drm_core_check_feature(dev, DRIVER_MODESET)) {
+                ret = drm_minor_alloc(dev, DRM_MINOR_CONTROL);
+                if (ret)
+                        goto err_minors;
+        }
+
+        if (drm_core_check_feature(dev, DRIVER_RENDER) && drm_rnodes) {
+                ret = drm_minor_alloc(dev, DRM_MINOR_RENDER);
+                if (ret)
+                        goto err_minors;
+        }
+
+        ret = drm_minor_alloc(dev, DRM_MINOR_LEGACY);
+        if (ret)
+                goto err_minors;
+
+        if (drm_ht_create(&dev->map_hash, 12))
+                goto err_minors;
 
         ret = drm_ctxbitmap_init(dev);
         if (ret) {
@@ -475,38 +621,71 @@ err_ctxbitmap:
         drm_ctxbitmap_cleanup(dev);
 err_ht:
         drm_ht_remove(&dev->map_hash);
+err_minors:
+        drm_minor_free(dev, DRM_MINOR_LEGACY);
+        drm_minor_free(dev, DRM_MINOR_RENDER);
+        drm_minor_free(dev, DRM_MINOR_CONTROL);
+        drm_fs_inode_free(dev->anon_inode);
 err_free:
+        mutex_destroy(&dev->master_mutex);
         kfree(dev);
         return NULL;
 }
 EXPORT_SYMBOL(drm_dev_alloc);
 
-/**
- * drm_dev_free - Free DRM device
- * @dev: DRM device to free
- *
- * Free a DRM device that has previously been allocated via drm_dev_alloc().
- * You must not use kfree() instead or you will leak memory.
- *
- * This must not be called once the device got registered. Use drm_put_dev()
- * instead, which then calls drm_dev_free().
- */
-void drm_dev_free(struct drm_device *dev)
+static void drm_dev_release(struct kref *ref)
 {
-        drm_put_minor(dev->control);
-        drm_put_minor(dev->render);
-        drm_put_minor(dev->primary);
+        struct drm_device *dev = container_of(ref, struct drm_device, ref);
 
         if (dev->driver->driver_features & DRIVER_GEM)
                 drm_gem_destroy(dev);
 
         drm_ctxbitmap_cleanup(dev);
         drm_ht_remove(&dev->map_hash);
+        drm_fs_inode_free(dev->anon_inode);
+
+        drm_minor_free(dev, DRM_MINOR_LEGACY);
+        drm_minor_free(dev, DRM_MINOR_RENDER);
+        drm_minor_free(dev, DRM_MINOR_CONTROL);
 
         kfree(dev->devname);
+
+        mutex_destroy(&dev->master_mutex);
         kfree(dev);
 }
-EXPORT_SYMBOL(drm_dev_free);
+
+/**
+ * drm_dev_ref - Take reference of a DRM device
+ * @dev: device to take reference of or NULL
+ *
+ * This increases the ref-count of @dev by one. You *must* already own a
+ * reference when calling this. Use drm_dev_unref() to drop this reference
+ * again.
+ *
+ * This function never fails. However, this function does not provide *any*
+ * guarantee whether the device is alive or running. It only provides a
+ * reference to the object and the memory associated with it.
+ */
+void drm_dev_ref(struct drm_device *dev)
+{
+        if (dev)
+                kref_get(&dev->ref);
+}
+EXPORT_SYMBOL(drm_dev_ref);
+
+/**
+ * drm_dev_unref - Drop reference of a DRM device
+ * @dev: device to drop reference of or NULL
+ *
+ * This decreases the ref-count of @dev by one. The device is destroyed if the
+ * ref-count drops to zero.
+ */
+void drm_dev_unref(struct drm_device *dev)
+{
+        if (dev)
+                kref_put(&dev->ref, drm_dev_release);
+}
+EXPORT_SYMBOL(drm_dev_unref);
 
 /**
  * drm_dev_register - Register DRM device
@@ -527,26 +706,22 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags)
 
         mutex_lock(&drm_global_mutex);
 
-        if (drm_core_check_feature(dev, DRIVER_MODESET)) {
-                ret = drm_get_minor(dev, &dev->control, DRM_MINOR_CONTROL);
-                if (ret)
-                        goto out_unlock;
-        }
+        ret = drm_minor_register(dev, DRM_MINOR_CONTROL);
+        if (ret)
+                goto err_minors;
 
-        if (drm_core_check_feature(dev, DRIVER_RENDER) && drm_rnodes) {
-                ret = drm_get_minor(dev, &dev->render, DRM_MINOR_RENDER);
-                if (ret)
-                        goto err_control_node;
-        }
+        ret = drm_minor_register(dev, DRM_MINOR_RENDER);
+        if (ret)
+                goto err_minors;
 
-        ret = drm_get_minor(dev, &dev->primary, DRM_MINOR_LEGACY);
+        ret = drm_minor_register(dev, DRM_MINOR_LEGACY);
         if (ret)
-                goto err_render_node;
+                goto err_minors;
 
         if (dev->driver->load) {
                 ret = dev->driver->load(dev, flags);
                 if (ret)
-                        goto err_primary_node;
+                        goto err_minors;
         }
 
         /* setup grouping for legacy outputs */
@@ -563,12 +738,10 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags)
 err_unload:
         if (dev->driver->unload)
                 dev->driver->unload(dev);
-err_primary_node:
-        drm_unplug_minor(dev->primary);
-err_render_node:
-        drm_unplug_minor(dev->render);
-err_control_node:
-        drm_unplug_minor(dev->control);
+err_minors:
+        drm_minor_unregister(dev, DRM_MINOR_LEGACY);
+        drm_minor_unregister(dev, DRM_MINOR_RENDER);
+        drm_minor_unregister(dev, DRM_MINOR_CONTROL);
 out_unlock:
         mutex_unlock(&drm_global_mutex);
         return ret;
@@ -581,7 +754,7 @@ EXPORT_SYMBOL(drm_dev_register);
  *
  * Unregister the DRM device from the system. This does the reverse of
  * drm_dev_register() but does not deallocate the device. The caller must call
- * drm_dev_free() to free all resources.
+ * drm_dev_unref() to drop their final reference.
  */
 void drm_dev_unregister(struct drm_device *dev)
 {
@@ -600,8 +773,8 @@ void drm_dev_unregister(struct drm_device *dev)
         list_for_each_entry_safe(r_list, list_temp, &dev->maplist, head)
                 drm_rmmap(dev, r_list->map);
 
-        drm_unplug_minor(dev->control);
-        drm_unplug_minor(dev->render);
-        drm_unplug_minor(dev->primary);
+        drm_minor_unregister(dev, DRM_MINOR_LEGACY);
+        drm_minor_unregister(dev, DRM_MINOR_RENDER);
+        drm_minor_unregister(dev, DRM_MINOR_CONTROL);
 }
 EXPORT_SYMBOL(drm_dev_unregister);
diff --git a/drivers/gpu/drm/drm_usb.c b/drivers/gpu/drm/drm_usb.c
index 0f8cb1ae7607..c3406aad2944 100644
--- a/drivers/gpu/drm/drm_usb.c
+++ b/drivers/gpu/drm/drm_usb.c
@@ -30,7 +30,7 @@ int drm_get_usb_dev(struct usb_interface *interface,
         return 0;
 
 err_free:
-        drm_dev_free(dev);
+        drm_dev_unref(dev);
         return ret;
 
 }
diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig
index 6e1a1a20cf6b..5bf5bca94f56 100644
--- a/drivers/gpu/drm/exynos/Kconfig
+++ b/drivers/gpu/drm/exynos/Kconfig
@@ -31,6 +31,30 @@ config DRM_EXYNOS_FIMD
         help
           Choose this option if you want to use Exynos FIMD for DRM.
 
+config DRM_EXYNOS_DPI
+        bool "EXYNOS DRM parallel output support"
+        depends on DRM_EXYNOS
+        select DRM_PANEL
+        default n
+        help
+          This enables support for Exynos parallel output.
+
+config DRM_EXYNOS_DSI
+        bool "EXYNOS DRM MIPI-DSI driver support"
+        depends on DRM_EXYNOS
+        select DRM_MIPI_DSI
+        select DRM_PANEL
+        default n
+        help
+          This enables support for Exynos MIPI-DSI device.
+
+config DRM_EXYNOS_DP
+        bool "EXYNOS DRM DP driver support"
+        depends on DRM_EXYNOS && ARCH_EXYNOS
+        default DRM_EXYNOS
+        help
+          This enables support for DP device.
+
 config DRM_EXYNOS_HDMI
         bool "Exynos DRM HDMI"
         depends on DRM_EXYNOS && !VIDEO_SAMSUNG_S5P_TV
diff --git a/drivers/gpu/drm/exynos/Makefile b/drivers/gpu/drm/exynos/Makefile
index 639b49e1ec05..33ae3652b8da 100644
--- a/drivers/gpu/drm/exynos/Makefile
+++ b/drivers/gpu/drm/exynos/Makefile
@@ -3,7 +3,7 @@
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
 ccflags-y := -Iinclude/drm -Idrivers/gpu/drm/exynos
-exynosdrm-y := exynos_drm_drv.o exynos_drm_encoder.o exynos_drm_connector.o \
+exynosdrm-y := exynos_drm_drv.o exynos_drm_encoder.o \
                 exynos_drm_crtc.o exynos_drm_fbdev.o exynos_drm_fb.o \
                 exynos_drm_buf.o exynos_drm_gem.o exynos_drm_core.o \
                 exynos_drm_plane.o
@@ -11,9 +11,10 @@ exynosdrm-y := exynos_drm_drv.o exynos_drm_encoder.o exynos_drm_connector.o \
 exynosdrm-$(CONFIG_DRM_EXYNOS_IOMMU) += exynos_drm_iommu.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_DMABUF) += exynos_drm_dmabuf.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_FIMD)     += exynos_drm_fimd.o
-exynosdrm-$(CONFIG_DRM_EXYNOS_HDMI)     += exynos_hdmi.o exynos_mixer.o \
-                                           exynos_ddc.o exynos_hdmiphy.o \
-                                           exynos_drm_hdmi.o
+exynosdrm-$(CONFIG_DRM_EXYNOS_DPI)      += exynos_drm_dpi.o
+exynosdrm-$(CONFIG_DRM_EXYNOS_DSI)      += exynos_drm_dsi.o
+exynosdrm-$(CONFIG_DRM_EXYNOS_DP)       += exynos_dp_core.o exynos_dp_reg.o
+exynosdrm-$(CONFIG_DRM_EXYNOS_HDMI)     += exynos_hdmi.o exynos_mixer.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_VIDI)     += exynos_drm_vidi.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_G2D)      += exynos_drm_g2d.o
 exynosdrm-$(CONFIG_DRM_EXYNOS_IPP)      += exynos_drm_ipp.o
diff --git a/drivers/video/exynos/exynos_dp_core.c b/drivers/gpu/drm/exynos/exynos_dp_core.c
index 5e1a71580051..aed533bbfd31 100644
--- a/drivers/video/exynos/exynos_dp_core.c
+++ b/drivers/gpu/drm/exynos/exynos_dp_core.c
@@ -12,7 +12,6 @@
 
 #include <linux/module.h>
 #include <linux/platform_device.h>
-#include <linux/slab.h>
 #include <linux/err.h>
 #include <linux/clk.h>
 #include <linux/io.h>
@@ -20,9 +19,25 @@
 #include <linux/delay.h>
 #include <linux/of.h>
 #include <linux/phy/phy.h>
+#include <video/of_display_timing.h>
+#include <video/of_videomode.h>
 
+#include <drm/drmP.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/bridge/ptn3460.h>
+
+#include "exynos_drm_drv.h"
 #include "exynos_dp_core.h"
 
+#define ctx_from_connector(c)   container_of(c, struct exynos_dp_device, \
+                                        connector)
+
+struct bridge_init {
+        struct i2c_client *client;
+        struct device_node *node;
+};
+
 static int exynos_dp_init_dp(struct exynos_dp_device *dp)
 {
         exynos_dp_reset(dp);
@@ -893,6 +908,214 @@ static void exynos_dp_hotplug(struct work_struct *work)
                 dev_err(dp->dev, "unable to config video\n");
 }
 
+static enum drm_connector_status exynos_dp_detect(
+                                struct drm_connector *connector, bool force)
+{
+        return connector_status_connected;
+}
+
+static void exynos_dp_connector_destroy(struct drm_connector *connector)
+{
+}
+
+static struct drm_connector_funcs exynos_dp_connector_funcs = {
+        .dpms = drm_helper_connector_dpms,
+        .fill_modes = drm_helper_probe_single_connector_modes,
+        .detect = exynos_dp_detect,
+        .destroy = exynos_dp_connector_destroy,
+};
+
+static int exynos_dp_get_modes(struct drm_connector *connector)
+{
+        struct exynos_dp_device *dp = ctx_from_connector(connector);
+        struct drm_display_mode *mode;
+
+        mode = drm_mode_create(connector->dev);
+        if (!mode) {
+                DRM_ERROR("failed to create a new display mode.\n");
+                return 0;
+        }
+
+        drm_display_mode_from_videomode(&dp->panel.vm, mode);
+        mode->width_mm = dp->panel.width_mm;
+        mode->height_mm = dp->panel.height_mm;
+        connector->display_info.width_mm = mode->width_mm;
+        connector->display_info.height_mm = mode->height_mm;
+
+        mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+        drm_mode_set_name(mode);
+        drm_mode_probed_add(connector, mode);
+
+        return 1;
+}
+
+static int exynos_dp_mode_valid(struct drm_connector *connector,
+                        struct drm_display_mode *mode)
+{
+        return MODE_OK;
+}
+
+static struct drm_encoder *exynos_dp_best_encoder(
+                        struct drm_connector *connector)
+{
+        struct exynos_dp_device *dp = ctx_from_connector(connector);
+
+        return dp->encoder;
+}
+
+static struct drm_connector_helper_funcs exynos_dp_connector_helper_funcs = {
+        .get_modes = exynos_dp_get_modes,
+        .mode_valid = exynos_dp_mode_valid,
+        .best_encoder = exynos_dp_best_encoder,
+};
+
+static int exynos_dp_initialize(struct exynos_drm_display *display,
+                                struct drm_device *drm_dev)
+{
+        struct exynos_dp_device *dp = display->ctx;
+
+        dp->drm_dev = drm_dev;
+
+        return 0;
+}
+
+static bool find_bridge(const char *compat, struct bridge_init *bridge)
+{
+        bridge->client = NULL;
+        bridge->node = of_find_compatible_node(NULL, NULL, compat);
+        if (!bridge->node)
+                return false;
+
+        bridge->client = of_find_i2c_device_by_node(bridge->node);
+        if (!bridge->client)
+                return false;
+
+        return true;
+}
+
+/* returns the number of bridges attached */
+static int exynos_drm_attach_lcd_bridge(struct drm_device *dev,
+                struct drm_encoder *encoder)
+{
+        struct bridge_init bridge;
+        int ret;
+
+        if (find_bridge("nxp,ptn3460", &bridge)) {
+                ret = ptn3460_init(dev, encoder, bridge.client, bridge.node);
+                if (!ret)
+                        return 1;
+        }
+        return 0;
+}
+
+static int exynos_dp_create_connector(struct exynos_drm_display *display,
+                                struct drm_encoder *encoder)
+{
+        struct exynos_dp_device *dp = display->ctx;
+        struct drm_connector *connector = &dp->connector;
+        int ret;
+
+        dp->encoder = encoder;
+
+        /* Pre-empt DP connector creation if there's a bridge */
+        ret = exynos_drm_attach_lcd_bridge(dp->drm_dev, encoder);
+        if (ret)
+                return 0;
+
+        connector->polled = DRM_CONNECTOR_POLL_HPD;
+
+        ret = drm_connector_init(dp->drm_dev, connector,
+                        &exynos_dp_connector_funcs, DRM_MODE_CONNECTOR_eDP);
+        if (ret) {
+                DRM_ERROR("Failed to initialize connector with drm\n");
+                return ret;
+        }
+
+        drm_connector_helper_add(connector, &exynos_dp_connector_helper_funcs);
+        drm_sysfs_connector_add(connector);
+        drm_mode_connector_attach_encoder(connector, encoder);
+
+        return 0;
+}
+
+static void exynos_dp_phy_init(struct exynos_dp_device *dp)
+{
+        if (dp->phy) {
+                phy_power_on(dp->phy);
+        } else if (dp->phy_addr) {
+                u32 reg;
+
+                reg = __raw_readl(dp->phy_addr);
+                reg |= dp->enable_mask;
+                __raw_writel(reg, dp->phy_addr);
+        }
+}
+
+static void exynos_dp_phy_exit(struct exynos_dp_device *dp)
+{
+        if (dp->phy) {
+                phy_power_off(dp->phy);
+        } else if (dp->phy_addr) {
+                u32 reg;
+
+                reg = __raw_readl(dp->phy_addr);
+                reg &= ~(dp->enable_mask);
+                __raw_writel(reg, dp->phy_addr);
+        }
+}
+
+static void exynos_dp_poweron(struct exynos_dp_device *dp)
+{
+        if (dp->dpms_mode == DRM_MODE_DPMS_ON)
+                return;
+
+        clk_prepare_enable(dp->clock);
+        exynos_dp_phy_init(dp);
+        exynos_dp_init_dp(dp);
+        enable_irq(dp->irq);
+}
+
+static void exynos_dp_poweroff(struct exynos_dp_device *dp)
+{
+        if (dp->dpms_mode != DRM_MODE_DPMS_ON)
+                return;
+
+        disable_irq(dp->irq);
+        flush_work(&dp->hotplug_work);
+        exynos_dp_phy_exit(dp);
+        clk_disable_unprepare(dp->clock);
+}
+
+static void exynos_dp_dpms(struct exynos_drm_display *display, int mode)
+{
+        struct exynos_dp_device *dp = display->ctx;
+
+        switch (mode) {
+        case DRM_MODE_DPMS_ON:
+                exynos_dp_poweron(dp);
+                break;
+        case DRM_MODE_DPMS_STANDBY:
+        case DRM_MODE_DPMS_SUSPEND:
+        case DRM_MODE_DPMS_OFF:
+                exynos_dp_poweroff(dp);
+                break;
+        default:
+                break;
+        };
+        dp->dpms_mode = mode;
+}
+
+static struct exynos_drm_display_ops exynos_dp_display_ops = {
+        .initialize = exynos_dp_initialize,
+        .create_connector = exynos_dp_create_connector,
+        .dpms = exynos_dp_dpms,
+};
+
+static struct exynos_drm_display exynos_dp_display = {
+        .type = EXYNOS_DISPLAY_TYPE_LCD,
+        .ops = &exynos_dp_display_ops,
+};
+
 static struct video_info *exynos_dp_dt_parse_pdata(struct device *dev)
 {
         struct device_node *dp_node = dev->of_node;
@@ -994,30 +1217,17 @@ err:
         return ret;
 }
 
-static void exynos_dp_phy_init(struct exynos_dp_device *dp)
-{
-        if (dp->phy) {
-                phy_power_on(dp->phy);
-        } else if (dp->phy_addr) {
-                u32 reg;
-
-                reg = __raw_readl(dp->phy_addr);
-                reg |= dp->enable_mask;
-                __raw_writel(reg, dp->phy_addr);
-        }
-}
-
-static void exynos_dp_phy_exit(struct exynos_dp_device *dp)
+static int exynos_dp_dt_parse_panel(struct exynos_dp_device *dp)
 {
-        if (dp->phy) {
-                phy_power_off(dp->phy);
-        } else if (dp->phy_addr) {
-                u32 reg;
+        int ret;
 
-                reg = __raw_readl(dp->phy_addr);
-                reg &= ~(dp->enable_mask);
-                __raw_writel(reg, dp->phy_addr);
+        ret = of_get_videomode(dp->dev->of_node, &dp->panel.vm,
+                        OF_USE_NATIVE_MODE);
+        if (ret) {
+                DRM_ERROR("failed: of_get_videomode() : %d\n", ret);
+                return ret;
         }
+        return 0;
 }
 
 static int exynos_dp_probe(struct platform_device *pdev)
@@ -1035,6 +1245,7 @@ static int exynos_dp_probe(struct platform_device *pdev)
         }
 
         dp->dev = &pdev->dev;
+        dp->dpms_mode = DRM_MODE_DPMS_OFF;
 
         dp->video_info = exynos_dp_dt_parse_pdata(&pdev->dev);
         if (IS_ERR(dp->video_info))
@@ -1044,6 +1255,10 @@ static int exynos_dp_probe(struct platform_device *pdev)
         if (ret)
                 return ret;
 
+        ret = exynos_dp_dt_parse_panel(dp);
+        if (ret)
+                return ret;
+
         dp->clock = devm_clk_get(&pdev->dev, "dp");
         if (IS_ERR(dp->clock)) {
                 dev_err(&pdev->dev, "failed to get clock\n");
@@ -1076,22 +1291,22 @@ static int exynos_dp_probe(struct platform_device *pdev)
                 dev_err(&pdev->dev, "failed to request irq\n");
                 return ret;
         }
+        disable_irq(dp->irq);
+
+        exynos_dp_display.ctx = dp;
 
-        platform_set_drvdata(pdev, dp);
+        platform_set_drvdata(pdev, &exynos_dp_display);
+        exynos_drm_display_register(&exynos_dp_display);
 
         return 0;
 }
 
 static int exynos_dp_remove(struct platform_device *pdev)
 {
-        struct exynos_dp_device *dp = platform_get_drvdata(pdev);
-
-        flush_work(&dp->hotplug_work);
-
-        exynos_dp_phy_exit(dp);
-
-        clk_disable_unprepare(dp->clock);
+        struct exynos_drm_display *display = platform_get_drvdata(pdev);
 
+        exynos_dp_dpms(display, DRM_MODE_DPMS_OFF);
+        exynos_drm_display_unregister(&exynos_dp_display);
 
         return 0;
 }
@@ -1099,31 +1314,19 @@ static int exynos_dp_remove(struct platform_device *pdev)
 #ifdef CONFIG_PM_SLEEP
 static int exynos_dp_suspend(struct device *dev)
 {
-        struct exynos_dp_device *dp = dev_get_drvdata(dev);
-
-        disable_irq(dp->irq);
-
-        flush_work(&dp->hotplug_work);
-
-        exynos_dp_phy_exit(dp);
-
-        clk_disable_unprepare(dp->clock);
+        struct platform_device *pdev = to_platform_device(dev);
+        struct exynos_drm_display *display = platform_get_drvdata(pdev);
 
+        exynos_dp_dpms(display, DRM_MODE_DPMS_OFF);
         return 0;
 }
 
 static int exynos_dp_resume(struct device *dev)
 {
-        struct exynos_dp_device *dp = dev_get_drvdata(dev);
-
-        exynos_dp_phy_init(dp);
-
-        clk_prepare_enable(dp->clock);
-
-        exynos_dp_init_dp(dp);
-
-        enable_irq(dp->irq);
+        struct platform_device *pdev = to_platform_device(dev);
+        struct exynos_drm_display *display = platform_get_drvdata(pdev);
 
+        exynos_dp_dpms(display, DRM_MODE_DPMS_ON);
         return 0;
 }
 #endif
@@ -1136,9 +1339,8 @@ static const struct of_device_id exynos_dp_match[] = {
         { .compatible = "samsung,exynos5-dp" },
         {},
 };
-MODULE_DEVICE_TABLE(of, exynos_dp_match);
 
-static struct platform_driver exynos_dp_driver = {
+struct platform_driver dp_driver = {
         .probe          = exynos_dp_probe,
         .remove         = exynos_dp_remove,
         .driver         = {
@@ -1149,8 +1351,6 @@ static struct platform_driver exynos_dp_driver = {
         },
 };
 
-module_platform_driver(exynos_dp_driver);
-
 MODULE_AUTHOR("Jingoo Han <jg1.han@samsung.com>");
 MODULE_DESCRIPTION("Samsung SoC DP Driver");
 MODULE_LICENSE("GPL");
diff --git a/drivers/video/exynos/exynos_dp_core.h b/drivers/gpu/drm/exynos/exynos_dp_core.h
index 607e36d0c147..d6a900d4ee40 100644
--- a/drivers/video/exynos/exynos_dp_core.h
+++ b/drivers/gpu/drm/exynos/exynos_dp_core.h
@@ -13,6 +13,9 @@
 #ifndef _EXYNOS_DP_CORE_H
 #define _EXYNOS_DP_CORE_H
 
+#include <drm/drm_crtc.h>
+#include <drm/exynos_drm.h>
+
 #define DP_TIMEOUT_LOOP_COUNT 100
 #define MAX_CR_LOOP 5
 #define MAX_EQ_LOOP 5
@@ -142,6 +145,9 @@ struct link_train {
 
 struct exynos_dp_device {
         struct device           *dev;
+        struct drm_device       *drm_dev;
+        struct drm_connector    connector;
+        struct drm_encoder      *encoder;
         struct clk              *clock;
         unsigned int            irq;
         void __iomem            *reg_base;
@@ -152,6 +158,9 @@ struct exynos_dp_device {
         struct link_train       link_train;
         struct work_struct      hotplug_work;
         struct phy              *phy;
+        int                     dpms_mode;
+
+        struct exynos_drm_panel_info panel;
 };
 
 /* exynos_dp_reg.c */
diff --git a/drivers/video/exynos/exynos_dp_reg.c b/drivers/gpu/drm/exynos/exynos_dp_reg.c
index b70da5052ff0..b70da5052ff0 100644
--- a/drivers/video/exynos/exynos_dp_reg.c
+++ b/drivers/gpu/drm/exynos/exynos_dp_reg.c
diff --git a/drivers/video/exynos/exynos_dp_reg.h b/drivers/gpu/drm/exynos/exynos_dp_reg.h
index 2e9bd0e0b9f2..2e9bd0e0b9f2 100644
--- a/drivers/video/exynos/exynos_dp_reg.h
+++ b/drivers/gpu/drm/exynos/exynos_dp_reg.h
diff --git a/drivers/gpu/drm/exynos/exynos_drm_connector.c b/drivers/gpu/drm/exynos/exynos_drm_connector.c
index e082efb2fece..9a16dbe121d1 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_connector.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_connector.c
@@ -23,27 +23,20 @@
                                 drm_connector)
 
 struct exynos_drm_connector {
-        struct drm_connector    drm_connector;
-        uint32_t                encoder_id;
-        struct exynos_drm_manager *manager;
-        uint32_t                dpms;
+        struct drm_connector            drm_connector;
+        uint32_t                        encoder_id;
+        struct exynos_drm_display       *display;
 };
 
 static int exynos_drm_connector_get_modes(struct drm_connector *connector)
 {
         struct exynos_drm_connector *exynos_connector =
                                         to_exynos_connector(connector);
-        struct exynos_drm_manager *manager = exynos_connector->manager;
-        struct exynos_drm_display_ops *display_ops = manager->display_ops;
+        struct exynos_drm_display *display = exynos_connector->display;
         struct edid *edid = NULL;
         unsigned int count = 0;
         int ret;
 
-        if (!display_ops) {
-                DRM_DEBUG_KMS("display_ops is null.\n");
-                return 0;
-        }
-
         /*
          * if get_edid() exists then get_edid() callback of hdmi side
          * is called to get edid data through i2c interface else
@@ -52,8 +45,8 @@ static int exynos_drm_connector_get_modes(struct drm_connector *connector)
          * P.S. in case of lcd panel, count is always 1 if success
          * because lcd panel has only one mode.
          */
-        if (display_ops->get_edid) {
-                edid = display_ops->get_edid(manager->dev, connector);
+        if (display->ops->get_edid) {
+                edid = display->ops->get_edid(display, connector);
                 if (IS_ERR_OR_NULL(edid)) {
                         ret = PTR_ERR(edid);
                         edid = NULL;
@@ -76,8 +69,8 @@ static int exynos_drm_connector_get_modes(struct drm_connector *connector)
                         return 0;
                 }
 
-                if (display_ops->get_panel)
-                        panel = display_ops->get_panel(manager->dev);
+                if (display->ops->get_panel)
+                        panel = display->ops->get_panel(display);
                 else {
                         drm_mode_destroy(connector->dev, mode);
                         return 0;
@@ -106,20 +99,20 @@ static int exynos_drm_connector_mode_valid(struct drm_connector *connector,
 {
         struct exynos_drm_connector *exynos_connector =
                                         to_exynos_connector(connector);
-        struct exynos_drm_manager *manager = exynos_connector->manager;
-        struct exynos_drm_display_ops *display_ops = manager->display_ops;
+        struct exynos_drm_display *display = exynos_connector->display;
         int ret = MODE_BAD;
 
         DRM_DEBUG_KMS("%s\n", __FILE__);
 
-        if (display_ops && display_ops->check_mode)
-                if (!display_ops->check_mode(manager->dev, mode))
+        if (display->ops->check_mode)
+                if (!display->ops->check_mode(display, mode))
                         ret = MODE_OK;
 
         return ret;
 }
 
-struct drm_encoder *exynos_drm_best_encoder(struct drm_connector *connector)
+static struct drm_encoder *exynos_drm_best_encoder(
+                struct drm_connector *connector)
 {
         struct drm_device *dev = connector->dev;
         struct exynos_drm_connector *exynos_connector =
@@ -146,48 +139,12 @@ static struct drm_connector_helper_funcs exynos_connector_helper_funcs = {
         .best_encoder   = exynos_drm_best_encoder,
 };
 
-void exynos_drm_display_power(struct drm_connector *connector, int mode)
-{
-        struct drm_encoder *encoder = exynos_drm_best_encoder(connector);
-        struct exynos_drm_connector *exynos_connector;
-        struct exynos_drm_manager *manager = exynos_drm_get_manager(encoder);
-        struct exynos_drm_display_ops *display_ops = manager->display_ops;
-
-        exynos_connector = to_exynos_connector(connector);
-
-        if (exynos_connector->dpms == mode) {
-                DRM_DEBUG_KMS("desired dpms mode is same as previous one.\n");
-                return;
-        }
-
-        if (display_ops && display_ops->power_on)
-                display_ops->power_on(manager->dev, mode);
-
-        exynos_connector->dpms = mode;
-}
-
-static void exynos_drm_connector_dpms(struct drm_connector *connector,
-                                        int mode)
-{
-        /*
-         * in case that drm_crtc_helper_set_mode() is called,
-         * encoder/crtc->funcs->dpms() will be just returned
-         * because they already were DRM_MODE_DPMS_ON so only
-         * exynos_drm_display_power() will be called.
-         */
-        drm_helper_connector_dpms(connector, mode);
-
-        exynos_drm_display_power(connector, mode);
-
-}
-
 static int exynos_drm_connector_fill_modes(struct drm_connector *connector,
                                 unsigned int max_width, unsigned int max_height)
 {
         struct exynos_drm_connector *exynos_connector =
                                         to_exynos_connector(connector);
-        struct exynos_drm_manager *manager = exynos_connector->manager;
-        struct exynos_drm_manager_ops *ops = manager->ops;
+        struct exynos_drm_display *display = exynos_connector->display;
         unsigned int width, height;
 
         width = max_width;
@@ -197,8 +154,8 @@ static int exynos_drm_connector_fill_modes(struct drm_connector *connector,
          * if specific driver want to find desired_mode using maxmum
          * resolution then get max width and height from that driver.
          */
-        if (ops && ops->get_max_resol)
-                ops->get_max_resol(manager->dev, &width, &height);
+        if (display->ops->get_max_resol)
+                display->ops->get_max_resol(display, &width, &height);
 
         return drm_helper_probe_single_connector_modes(connector, width,
                                                         height);
@@ -210,13 +167,11 @@ exynos_drm_connector_detect(struct drm_connector *connector, bool force)
 {
         struct exynos_drm_connector *exynos_connector =
                                         to_exynos_connector(connector);
-        struct exynos_drm_manager *manager = exynos_connector->manager;
-        struct exynos_drm_display_ops *display_ops =
-                                        manager->display_ops;
+        struct exynos_drm_display *display = exynos_connector->display;
         enum drm_connector_status status = connector_status_disconnected;
 
-        if (display_ops && display_ops->is_connected) {
-                if (display_ops->is_connected(manager->dev))
+        if (display->ops->is_connected) {
+                if (display->ops->is_connected(display))
                         status = connector_status_connected;
                 else
                         status = connector_status_disconnected;
@@ -236,7 +191,7 @@ static void exynos_drm_connector_destroy(struct drm_connector *connector)
 }
 
 static struct drm_connector_funcs exynos_connector_funcs = {
-        .dpms           = exynos_drm_connector_dpms,
+        .dpms           = drm_helper_connector_dpms,
         .fill_modes     = exynos_drm_connector_fill_modes,
         .detect         = exynos_drm_connector_detect,
         .destroy        = exynos_drm_connector_destroy,
@@ -246,7 +201,7 @@ struct drm_connector *exynos_drm_connector_create(struct drm_device *dev,
                                                    struct drm_encoder *encoder)
 {
         struct exynos_drm_connector *exynos_connector;
-        struct exynos_drm_manager *manager = exynos_drm_get_manager(encoder);
+        struct exynos_drm_display *display = exynos_drm_get_display(encoder);
         struct drm_connector *connector;
         int type;
         int err;
@@ -257,7 +212,7 @@ struct drm_connector *exynos_drm_connector_create(struct drm_device *dev,
 
         connector = &exynos_connector->drm_connector;
 
-        switch (manager->display_ops->type) {
+        switch (display->type) {
         case EXYNOS_DISPLAY_TYPE_HDMI:
                 type = DRM_MODE_CONNECTOR_HDMIA;
                 connector->interlace_allowed = true;
@@ -280,8 +235,7 @@ struct drm_connector *exynos_drm_connector_create(struct drm_device *dev,
                 goto err_connector;
 
         exynos_connector->encoder_id = encoder->base.id;
-        exynos_connector->manager = manager;
-        exynos_connector->dpms = DRM_MODE_DPMS_OFF;
+        exynos_connector->display = display;
         connector->dpms = DRM_MODE_DPMS_OFF;
         connector->encoder = encoder;
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_connector.h b/drivers/gpu/drm/exynos/exynos_drm_connector.h
index 547c6b590357..4eb20d78379a 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_connector.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_connector.h
@@ -17,8 +17,4 @@
 struct drm_connector *exynos_drm_connector_create(struct drm_device *dev,
                                                    struct drm_encoder *encoder);
 
-struct drm_encoder *exynos_drm_best_encoder(struct drm_connector *connector);
-
-void exynos_drm_display_power(struct drm_connector *connector, int mode);
-
 #endif
diff --git a/drivers/gpu/drm/exynos/exynos_drm_core.c b/drivers/gpu/drm/exynos/exynos_drm_core.c
index 1bef6dc77478..0e9e06ce36b8 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_core.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_core.c
@@ -14,43 +14,42 @@
 
 #include <drm/drmP.h>
 #include "exynos_drm_drv.h"
+#include "exynos_drm_crtc.h"
 #include "exynos_drm_encoder.h"
-#include "exynos_drm_connector.h"
 #include "exynos_drm_fbdev.h"
 
 static LIST_HEAD(exynos_drm_subdrv_list);
+static LIST_HEAD(exynos_drm_manager_list);
+static LIST_HEAD(exynos_drm_display_list);
 
 static int exynos_drm_create_enc_conn(struct drm_device *dev,
-                                        struct exynos_drm_subdrv *subdrv)
+                                        struct exynos_drm_display *display)
 {
         struct drm_encoder *encoder;
-        struct drm_connector *connector;
+        struct exynos_drm_manager *manager;
         int ret;
+        unsigned long possible_crtcs = 0;
 
-        subdrv->manager->dev = subdrv->dev;
+        /* Find possible crtcs for this display */
+        list_for_each_entry(manager, &exynos_drm_manager_list, list)
+                if (manager->type == display->type)
+                        possible_crtcs |= 1 << manager->pipe;
 
         /* create and initialize a encoder for this sub driver. */
-        encoder = exynos_drm_encoder_create(dev, subdrv->manager,
-                        (1 << MAX_CRTC) - 1);
+        encoder = exynos_drm_encoder_create(dev, display, possible_crtcs);
         if (!encoder) {
                 DRM_ERROR("failed to create encoder\n");
                 return -EFAULT;
         }
 
-        /*
-         * create and initialize a connector for this sub driver and
-         * attach the encoder created above to the connector.
-         */
-        connector = exynos_drm_connector_create(dev, encoder);
-        if (!connector) {
-                DRM_ERROR("failed to create connector\n");
-                ret = -EFAULT;
+        display->encoder = encoder;
+
+        ret = display->ops->create_connector(display, encoder);
+        if (ret) {
+                DRM_ERROR("failed to create connector ret = %d\n", ret);
                 goto err_destroy_encoder;
         }
 
-        subdrv->encoder = encoder;
-        subdrv->connector = connector;
-
         return 0;
 
 err_destroy_encoder:
@@ -58,21 +57,6 @@ err_destroy_encoder:
         return ret;
 }
 
-static void exynos_drm_destroy_enc_conn(struct exynos_drm_subdrv *subdrv)
-{
-        if (subdrv->encoder) {
-                struct drm_encoder *encoder = subdrv->encoder;
-                encoder->funcs->destroy(encoder);
-                subdrv->encoder = NULL;
-        }
-
-        if (subdrv->connector) {
-                struct drm_connector *connector = subdrv->connector;
-                connector->funcs->destroy(connector);
-                subdrv->connector = NULL;
-        }
-}
-
 static int exynos_drm_subdrv_probe(struct drm_device *dev,
                                         struct exynos_drm_subdrv *subdrv)
 {
@@ -104,10 +88,98 @@ static void exynos_drm_subdrv_remove(struct drm_device *dev,
                 subdrv->remove(dev, subdrv->dev);
 }
 
+int exynos_drm_initialize_managers(struct drm_device *dev)
+{
+        struct exynos_drm_manager *manager, *n;
+        int ret, pipe = 0;
+
+        list_for_each_entry(manager, &exynos_drm_manager_list, list) {
+                if (manager->ops->initialize) {
+                        ret = manager->ops->initialize(manager, dev, pipe);
+                        if (ret) {
+                                DRM_ERROR("Mgr init [%d] failed with %d\n",
+                                                manager->type, ret);
+                                goto err;
+                        }
+                }
+
+                manager->drm_dev = dev;
+                manager->pipe = pipe++;
+
+                ret = exynos_drm_crtc_create(manager);
+                if (ret) {
+                        DRM_ERROR("CRTC create [%d] failed with %d\n",
+                                        manager->type, ret);
+                        goto err;
+                }
+        }
+        return 0;
+
+err:
+        list_for_each_entry_safe(manager, n, &exynos_drm_manager_list, list) {
+                if (pipe-- > 0)
+                        exynos_drm_manager_unregister(manager);
+                else
+                        list_del(&manager->list);
+        }
+        return ret;
+}
+
+void exynos_drm_remove_managers(struct drm_device *dev)
+{
+        struct exynos_drm_manager *manager, *n;
+
+        list_for_each_entry_safe(manager, n, &exynos_drm_manager_list, list)
+                exynos_drm_manager_unregister(manager);
+}
+
+int exynos_drm_initialize_displays(struct drm_device *dev)
+{
+        struct exynos_drm_display *display, *n;
+        int ret, initialized = 0;
+
+        list_for_each_entry(display, &exynos_drm_display_list, list) {
+                if (display->ops->initialize) {
+                        ret = display->ops->initialize(display, dev);
+                        if (ret) {
+                                DRM_ERROR("Display init [%d] failed with %d\n",
+                                                display->type, ret);
+                                goto err;
+                        }
+                }
+
+                initialized++;
+
+                ret = exynos_drm_create_enc_conn(dev, display);
+                if (ret) {
+                        DRM_ERROR("Encoder create [%d] failed with %d\n",
+                                        display->type, ret);
+                        goto err;
+                }
+        }
+        return 0;
+
+err:
+        list_for_each_entry_safe(display, n, &exynos_drm_display_list, list) {
+                if (initialized-- > 0)
+                        exynos_drm_display_unregister(display);
+                else
+                        list_del(&display->list);
+        }
+        return ret;
+}
+
+void exynos_drm_remove_displays(struct drm_device *dev)
+{
+        struct exynos_drm_display *display, *n;
+
+        list_for_each_entry_safe(display, n, &exynos_drm_display_list, list)
+                exynos_drm_display_unregister(display);
+}
+
 int exynos_drm_device_register(struct drm_device *dev)
 {
         struct exynos_drm_subdrv *subdrv, *n;
-        unsigned int fine_cnt = 0;
         int err;
 
         if (!dev)
@@ -120,30 +192,8 @@ int exynos_drm_device_register(struct drm_device *dev)
                         list_del(&subdrv->list);
                         continue;
                 }
-
-                /*
-                 * if manager is null then it means that this sub driver
-                 * doesn't need encoder and connector.
-                 */
-                if (!subdrv->manager) {
-                        fine_cnt++;
-                        continue;
-                }
-
-                err = exynos_drm_create_enc_conn(dev, subdrv);
-                if (err) {
-                        DRM_DEBUG("failed to create encoder and connector.\n");
-                        exynos_drm_subdrv_remove(dev, subdrv);
-                        list_del(&subdrv->list);
-                        continue;
-                }
-
-                fine_cnt++;
         }
 
-        if (!fine_cnt)
-                return -EINVAL;
-
         return 0;
 }
 EXPORT_SYMBOL_GPL(exynos_drm_device_register);
@@ -159,13 +209,44 @@ int exynos_drm_device_unregister(struct drm_device *dev)
 
         list_for_each_entry(subdrv, &exynos_drm_subdrv_list, list) {
                 exynos_drm_subdrv_remove(dev, subdrv);
-                exynos_drm_destroy_enc_conn(subdrv);
         }
 
         return 0;
 }
 EXPORT_SYMBOL_GPL(exynos_drm_device_unregister);
 
+int exynos_drm_manager_register(struct exynos_drm_manager *manager)
+{
+        BUG_ON(!manager->ops);
+        list_add_tail(&manager->list, &exynos_drm_manager_list);
+        return 0;
+}
+
+int exynos_drm_manager_unregister(struct exynos_drm_manager *manager)
+{
+        if (manager->ops->remove)
+                manager->ops->remove(manager);
+
+        list_del(&manager->list);
+        return 0;
+}
+
+int exynos_drm_display_register(struct exynos_drm_display *display)
+{
+        BUG_ON(!display->ops);
+        list_add_tail(&display->list, &exynos_drm_display_list);
+        return 0;
+}
+
+int exynos_drm_display_unregister(struct exynos_drm_display *display)
+{
+        if (display->ops->remove)
+                display->ops->remove(display);
+
+        list_del(&display->list);
+        return 0;
+}
+
 int exynos_drm_subdrv_register(struct exynos_drm_subdrv *subdrv)
 {
         if (!subdrv)
diff --git a/drivers/gpu/drm/exynos/exynos_drm_crtc.c b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
index 6f3400f3978a..e930d4fe29c7 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_crtc.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_crtc.c
@@ -33,6 +33,7 @@ enum exynos_crtc_mode {
  *
  * @drm_crtc: crtc object.
  * @drm_plane: pointer of private plane object for this crtc
+ * @manager: the manager associated with this crtc
  * @pipe: a crtc index created at load() with a new crtc object creation
  *      and the crtc object would be set to private->crtc array
  *      to get a crtc object corresponding to this pipe from private->crtc
@@ -46,6 +47,7 @@ enum exynos_crtc_mode {
 struct exynos_drm_crtc {
         struct drm_crtc                 drm_crtc;
         struct drm_plane                *plane;
+        struct exynos_drm_manager       *manager;
         unsigned int                    pipe;
         unsigned int                    dpms;
         enum exynos_crtc_mode           mode;
@@ -56,6 +58,7 @@ struct exynos_drm_crtc {
 static void exynos_drm_crtc_dpms(struct drm_crtc *crtc, int mode)
 {
         struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
+        struct exynos_drm_manager *manager = exynos_crtc->manager;
 
         DRM_DEBUG_KMS("crtc[%d] mode[%d]\n", crtc->base.id, mode);
 
@@ -71,7 +74,9 @@ static void exynos_drm_crtc_dpms(struct drm_crtc *crtc, int mode)
                 drm_vblank_off(crtc->dev, exynos_crtc->pipe);
         }
 
-        exynos_drm_fn_encoder(crtc, &mode, exynos_drm_encoder_crtc_dpms);
+        if (manager->ops->dpms)
+                manager->ops->dpms(manager, mode);
+
         exynos_crtc->dpms = mode;
 }
 
@@ -83,9 +88,15 @@ static void exynos_drm_crtc_prepare(struct drm_crtc *crtc)
 static void exynos_drm_crtc_commit(struct drm_crtc *crtc)
 {
         struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
+        struct exynos_drm_manager *manager = exynos_crtc->manager;
 
         exynos_drm_crtc_dpms(crtc, DRM_MODE_DPMS_ON);
+
         exynos_plane_commit(exynos_crtc->plane);
+
+        if (manager->ops->commit)
+                manager->ops->commit(manager);
+
         exynos_plane_dpms(exynos_crtc->plane, DRM_MODE_DPMS_ON);
 }
 
@@ -94,7 +105,12 @@ exynos_drm_crtc_mode_fixup(struct drm_crtc *crtc,
                             const struct drm_display_mode *mode,
                             struct drm_display_mode *adjusted_mode)
 {
-        /* drm framework doesn't check NULL */
+        struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
+        struct exynos_drm_manager *manager = exynos_crtc->manager;
+
+        if (manager->ops->mode_fixup)
+                return manager->ops->mode_fixup(manager, mode, adjusted_mode);
+
         return true;
 }
 
@@ -104,10 +120,10 @@ exynos_drm_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode,
                           struct drm_framebuffer *old_fb)
 {
         struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
+        struct exynos_drm_manager *manager = exynos_crtc->manager;
         struct drm_plane *plane = exynos_crtc->plane;
         unsigned int crtc_w;
         unsigned int crtc_h;
-        int pipe = exynos_crtc->pipe;
         int ret;
 
         /*
@@ -116,18 +132,19 @@ exynos_drm_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode,
          */
         memcpy(&crtc->mode, adjusted_mode, sizeof(*adjusted_mode));
 
-        crtc_w = crtc->fb->width - x;
-        crtc_h = crtc->fb->height - y;
+        crtc_w = crtc->primary->fb->width - x;
+        crtc_h = crtc->primary->fb->height - y;
+
+        if (manager->ops->mode_set)
+                manager->ops->mode_set(manager, &crtc->mode);
 
-        ret = exynos_plane_mode_set(plane, crtc, crtc->fb, 0, 0, crtc_w, crtc_h,
+        ret = exynos_plane_mode_set(plane, crtc, crtc->primary->fb, 0, 0, crtc_w, crtc_h,
                                     x, y, crtc_w, crtc_h);
         if (ret)
                 return ret;
 
         plane->crtc = crtc;
-        plane->fb = crtc->fb;
-
-        exynos_drm_fn_encoder(crtc, &pipe, exynos_drm_encoder_crtc_pipe);
+        plane->fb = crtc->primary->fb;
 
         return 0;
 }
@@ -147,10 +164,10 @@ static int exynos_drm_crtc_mode_set_commit(struct drm_crtc *crtc, int x, int y,
                 return -EPERM;
         }
 
-        crtc_w = crtc->fb->width - x;
-        crtc_h = crtc->fb->height - y;
+        crtc_w = crtc->primary->fb->width - x;
+        crtc_h = crtc->primary->fb->height - y;
 
-        ret = exynos_plane_mode_set(plane, crtc, crtc->fb, 0, 0, crtc_w, crtc_h,
+        ret = exynos_plane_mode_set(plane, crtc, crtc->primary->fb, 0, 0, crtc_w, crtc_h,
                                     x, y, crtc_w, crtc_h);
         if (ret)
                 return ret;
@@ -168,10 +185,19 @@ static int exynos_drm_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
 
 static void exynos_drm_crtc_disable(struct drm_crtc *crtc)
 {
-        struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
+        struct drm_plane *plane;
+        int ret;
 
-        exynos_plane_dpms(exynos_crtc->plane, DRM_MODE_DPMS_OFF);
         exynos_drm_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
+
+        drm_for_each_legacy_plane(plane, &crtc->dev->mode_config.plane_list) {
+                if (plane->crtc != crtc)
+                        continue;
+
+                ret = plane->funcs->disable_plane(plane);
+                if (ret)
+                        DRM_ERROR("Failed to disable plane %d\n", ret);
+        }
 }
 
 static struct drm_crtc_helper_funcs exynos_crtc_helper_funcs = {
@@ -192,7 +218,7 @@ static int exynos_drm_crtc_page_flip(struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         struct exynos_drm_private *dev_priv = dev->dev_private;
         struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(crtc);
-        struct drm_framebuffer *old_fb = crtc->fb;
+        struct drm_framebuffer *old_fb = crtc->primary->fb;
         int ret = -EINVAL;
 
         /* when the page flip is requested, crtc's dpms should be on */
@@ -223,11 +249,11 @@ static int exynos_drm_crtc_page_flip(struct drm_crtc *crtc,
                 atomic_set(&exynos_crtc->pending_flip, 1);
                 spin_unlock_irq(&dev->event_lock);
 
-                crtc->fb = fb;
+                crtc->primary->fb = fb;
                 ret = exynos_drm_crtc_mode_set_commit(crtc, crtc->x, crtc->y,
                                                     NULL);
                 if (ret) {
-                        crtc->fb = old_fb;
+                        crtc->primary->fb = old_fb;
 
                         spin_lock_irq(&dev->event_lock);
                         drm_vblank_put(dev, exynos_crtc->pipe);
@@ -318,21 +344,24 @@ static void exynos_drm_crtc_attach_mode_property(struct drm_crtc *crtc)
         drm_object_attach_property(&crtc->base, prop, 0);
 }
 
-int exynos_drm_crtc_create(struct drm_device *dev, unsigned int nr)
+int exynos_drm_crtc_create(struct exynos_drm_manager *manager)
 {
         struct exynos_drm_crtc *exynos_crtc;
-        struct exynos_drm_private *private = dev->dev_private;
+        struct exynos_drm_private *private = manager->drm_dev->dev_private;
         struct drm_crtc *crtc;
 
         exynos_crtc = kzalloc(sizeof(*exynos_crtc), GFP_KERNEL);
         if (!exynos_crtc)
                 return -ENOMEM;
 
-        exynos_crtc->pipe = nr;
-        exynos_crtc->dpms = DRM_MODE_DPMS_OFF;
         init_waitqueue_head(&exynos_crtc->pending_flip_queue);
         atomic_set(&exynos_crtc->pending_flip, 0);
-        exynos_crtc->plane = exynos_plane_init(dev, 1 << nr, true);
+
+        exynos_crtc->dpms = DRM_MODE_DPMS_OFF;
+        exynos_crtc->manager = manager;
+        exynos_crtc->pipe = manager->pipe;
+        exynos_crtc->plane = exynos_plane_init(manager->drm_dev,
+                                1 << manager->pipe, true);
         if (!exynos_crtc->plane) {
                 kfree(exynos_crtc);
                 return -ENOMEM;
@@ -340,9 +369,9 @@ int exynos_drm_crtc_create(struct drm_device *dev, unsigned int nr)
 
         crtc = &exynos_crtc->drm_crtc;
 
-        private->crtc[nr] = crtc;
+        private->crtc[manager->pipe] = crtc;
 
-        drm_crtc_init(dev, crtc, &exynos_crtc_funcs);
+        drm_crtc_init(manager->drm_dev, crtc, &exynos_crtc_funcs);
         drm_crtc_helper_add(crtc, &exynos_crtc_helper_funcs);
 
         exynos_drm_crtc_attach_mode_property(crtc);
@@ -350,39 +379,41 @@ int exynos_drm_crtc_create(struct drm_device *dev, unsigned int nr)
         return 0;
 }
 
-int exynos_drm_crtc_enable_vblank(struct drm_device *dev, int crtc)
+int exynos_drm_crtc_enable_vblank(struct drm_device *dev, int pipe)
 {
         struct exynos_drm_private *private = dev->dev_private;
         struct exynos_drm_crtc *exynos_crtc =
-                to_exynos_crtc(private->crtc[crtc]);
+                to_exynos_crtc(private->crtc[pipe]);
+        struct exynos_drm_manager *manager = exynos_crtc->manager;
 
         if (exynos_crtc->dpms != DRM_MODE_DPMS_ON)
                 return -EPERM;
 
-        exynos_drm_fn_encoder(private->crtc[crtc], &crtc,
-                        exynos_drm_enable_vblank);
+        if (manager->ops->enable_vblank)
+                manager->ops->enable_vblank(manager);
 
         return 0;
 }
 
-void exynos_drm_crtc_disable_vblank(struct drm_device *dev, int crtc)
+void exynos_drm_crtc_disable_vblank(struct drm_device *dev, int pipe)
 {
         struct exynos_drm_private *private = dev->dev_private;
         struct exynos_drm_crtc *exynos_crtc =
-                to_exynos_crtc(private->crtc[crtc]);
+                to_exynos_crtc(private->crtc[pipe]);
+        struct exynos_drm_manager *manager = exynos_crtc->manager;
 
         if (exynos_crtc->dpms != DRM_MODE_DPMS_ON)
                 return;
 
-        exynos_drm_fn_encoder(private->crtc[crtc], &crtc,
-                        exynos_drm_disable_vblank);
+        if (manager->ops->disable_vblank)
+                manager->ops->disable_vblank(manager);
 }
 
-void exynos_drm_crtc_finish_pageflip(struct drm_device *dev, int crtc)
+void exynos_drm_crtc_finish_pageflip(struct drm_device *dev, int pipe)
 {
         struct exynos_drm_private *dev_priv = dev->dev_private;
         struct drm_pending_vblank_event *e, *t;
-        struct drm_crtc *drm_crtc = dev_priv->crtc[crtc];
+        struct drm_crtc *drm_crtc = dev_priv->crtc[pipe];
         struct exynos_drm_crtc *exynos_crtc = to_exynos_crtc(drm_crtc);
         unsigned long flags;
 
@@ -391,15 +422,71 @@ void exynos_drm_crtc_finish_pageflip(struct drm_device *dev, int crtc)
         list_for_each_entry_safe(e, t, &dev_priv->pageflip_event_list,
                         base.link) {
                 /* if event's pipe isn't same as crtc then ignore it. */
-                if (crtc != e->pipe)
+                if (pipe != e->pipe)
                         continue;
 
                 list_del(&e->base.link);
                 drm_send_vblank_event(dev, -1, e);
-                drm_vblank_put(dev, crtc);
+                drm_vblank_put(dev, pipe);
                 atomic_set(&exynos_crtc->pending_flip, 0);
                 wake_up(&exynos_crtc->pending_flip_queue);
         }
 
         spin_unlock_irqrestore(&dev->event_lock, flags);
 }
+
+void exynos_drm_crtc_plane_mode_set(struct drm_crtc *crtc,
+                        struct exynos_drm_overlay *overlay)
+{
+        struct exynos_drm_manager *manager = to_exynos_crtc(crtc)->manager;
+
+        if (manager->ops->win_mode_set)
+                manager->ops->win_mode_set(manager, overlay);
+}
+
+void exynos_drm_crtc_plane_commit(struct drm_crtc *crtc, int zpos)
+{
+        struct exynos_drm_manager *manager = to_exynos_crtc(crtc)->manager;
+
+        if (manager->ops->win_commit)
+                manager->ops->win_commit(manager, zpos);
+}
+
+void exynos_drm_crtc_plane_enable(struct drm_crtc *crtc, int zpos)
+{
+        struct exynos_drm_manager *manager = to_exynos_crtc(crtc)->manager;
+
+        if (manager->ops->win_enable)
+                manager->ops->win_enable(manager, zpos);
+}
+
+void exynos_drm_crtc_plane_disable(struct drm_crtc *crtc, int zpos)
+{
+        struct exynos_drm_manager *manager = to_exynos_crtc(crtc)->manager;
+
+        if (manager->ops->win_disable)
+                manager->ops->win_disable(manager, zpos);
+}
+
+void exynos_drm_crtc_complete_scanout(struct drm_framebuffer *fb)
+{
+        struct exynos_drm_manager *manager;
+        struct drm_device *dev = fb->dev;
+        struct drm_crtc *crtc;
+
+        /*
+         * make sure that overlay data are updated to real hardware
+         * for all encoders.
+         */
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                manager = to_exynos_crtc(crtc)->manager;
+
+                /*
+                 * wait for vblank interrupt
+                 * - this makes sure that overlay data are updated to
+                 *      real hardware.
+                 */
+                if (manager->ops->wait_for_vblank)
+                        manager->ops->wait_for_vblank(manager);
+        }
+}
diff --git a/drivers/gpu/drm/exynos/exynos_drm_crtc.h b/drivers/gpu/drm/exynos/exynos_drm_crtc.h
index 3e197e6ae7d9..c27b66cc5d24 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_crtc.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_crtc.h
@@ -15,9 +15,21 @@
 #ifndef _EXYNOS_DRM_CRTC_H_
 #define _EXYNOS_DRM_CRTC_H_
 
-int exynos_drm_crtc_create(struct drm_device *dev, unsigned int nr);
-int exynos_drm_crtc_enable_vblank(struct drm_device *dev, int crtc);
-void exynos_drm_crtc_disable_vblank(struct drm_device *dev, int crtc);
-void exynos_drm_crtc_finish_pageflip(struct drm_device *dev, int crtc);
+struct drm_device;
+struct drm_crtc;
+struct exynos_drm_manager;
+struct exynos_drm_overlay;
+
+int exynos_drm_crtc_create(struct exynos_drm_manager *manager);
+int exynos_drm_crtc_enable_vblank(struct drm_device *dev, int pipe);
+void exynos_drm_crtc_disable_vblank(struct drm_device *dev, int pipe);
+void exynos_drm_crtc_finish_pageflip(struct drm_device *dev, int pipe);
+void exynos_drm_crtc_complete_scanout(struct drm_framebuffer *fb);
+
+void exynos_drm_crtc_plane_mode_set(struct drm_crtc *crtc,
+                        struct exynos_drm_overlay *overlay);
+void exynos_drm_crtc_plane_commit(struct drm_crtc *crtc, int zpos);
+void exynos_drm_crtc_plane_enable(struct drm_crtc *crtc, int zpos);
+void exynos_drm_crtc_plane_disable(struct drm_crtc *crtc, int zpos);
 
 #endif
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dpi.c b/drivers/gpu/drm/exynos/exynos_drm_dpi.c
new file mode 100644
index 000000000000..2b09c7c0bfcc
--- /dev/null
+++ b/drivers/gpu/drm/exynos/exynos_drm_dpi.c
@@ -0,0 +1,339 @@
+/*
+ * Exynos DRM Parallel output support.
+ *
+ * Copyright (c) 2014 Samsung Electronics Co., Ltd
+ *
+ * Contacts: Andrzej Hajda <a.hajda@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+*/
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_panel.h>
+
+#include <linux/regulator/consumer.h>
+
+#include <video/of_videomode.h>
+#include <video/videomode.h>
+
+#include "exynos_drm_drv.h"
+
+struct exynos_dpi {
+        struct device *dev;
+        struct device_node *panel_node;
+
+        struct drm_panel *panel;
+        struct drm_connector connector;
+        struct drm_encoder *encoder;
+
+        struct videomode *vm;
+        int dpms_mode;
+};
+
+#define connector_to_dpi(c) container_of(c, struct exynos_dpi, connector)
+
+static enum drm_connector_status
+exynos_dpi_detect(struct drm_connector *connector, bool force)
+{
+        struct exynos_dpi *ctx = connector_to_dpi(connector);
+
+        /* panels supported only by boot-loader are always connected */
+        if (!ctx->panel_node)
+                return connector_status_connected;
+
+        if (!ctx->panel) {
+                ctx->panel = of_drm_find_panel(ctx->panel_node);
+                if (ctx->panel)
+                        drm_panel_attach(ctx->panel, &ctx->connector);
+        }
+
+        if (ctx->panel)
+                return connector_status_connected;
+
+        return connector_status_disconnected;
+}
+
+static void exynos_dpi_connector_destroy(struct drm_connector *connector)
+{
+        drm_sysfs_connector_remove(connector);
+        drm_connector_cleanup(connector);
+}
+
+static struct drm_connector_funcs exynos_dpi_connector_funcs = {
+        .dpms = drm_helper_connector_dpms,
+        .detect = exynos_dpi_detect,
+        .fill_modes = drm_helper_probe_single_connector_modes,
+        .destroy = exynos_dpi_connector_destroy,
+};
+
+static int exynos_dpi_get_modes(struct drm_connector *connector)
+{
+        struct exynos_dpi *ctx = connector_to_dpi(connector);
+
+        /* fimd timings gets precedence over panel modes */
+        if (ctx->vm) {
+                struct drm_display_mode *mode;
+
+                mode = drm_mode_create(connector->dev);
+                if (!mode) {
+                        DRM_ERROR("failed to create a new display mode\n");
+                        return 0;
+                }
+                drm_display_mode_from_videomode(ctx->vm, mode);
+                mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+                drm_mode_probed_add(connector, mode);
+                return 1;
+        }
+
+        if (ctx->panel)
+                return ctx->panel->funcs->get_modes(ctx->panel);
+
+        return 0;
+}
+
+static int exynos_dpi_mode_valid(struct drm_connector *connector,
+                                 struct drm_display_mode *mode)
+{
+        return MODE_OK;
+}
+
+static struct drm_encoder *
+exynos_dpi_best_encoder(struct drm_connector *connector)
+{
+        struct exynos_dpi *ctx = connector_to_dpi(connector);
+
+        return ctx->encoder;
+}
+
+static struct drm_connector_helper_funcs exynos_dpi_connector_helper_funcs = {
+        .get_modes = exynos_dpi_get_modes,
+        .mode_valid = exynos_dpi_mode_valid,
+        .best_encoder = exynos_dpi_best_encoder,
+};
+
+static int exynos_dpi_create_connector(struct exynos_drm_display *display,
+                                       struct drm_encoder *encoder)
+{
+        struct exynos_dpi *ctx = display->ctx;
+        struct drm_connector *connector = &ctx->connector;
+        int ret;
+
+        ctx->encoder = encoder;
+
+        if (ctx->panel_node)
+                connector->polled = DRM_CONNECTOR_POLL_CONNECT;
+        else
+                connector->polled = DRM_CONNECTOR_POLL_HPD;
+
+        ret = drm_connector_init(encoder->dev, connector,
+                                 &exynos_dpi_connector_funcs,
+                                 DRM_MODE_CONNECTOR_VGA);
+        if (ret) {
+                DRM_ERROR("failed to initialize connector with drm\n");
+                return ret;
+        }
+
+        drm_connector_helper_add(connector, &exynos_dpi_connector_helper_funcs);
+        drm_sysfs_connector_add(connector);
+        drm_mode_connector_attach_encoder(connector, encoder);
+
+        return 0;
+}
+
+static void exynos_dpi_poweron(struct exynos_dpi *ctx)
+{
+        if (ctx->panel)
+                drm_panel_enable(ctx->panel);
+}
+
+static void exynos_dpi_poweroff(struct exynos_dpi *ctx)
+{
+        if (ctx->panel)
+                drm_panel_disable(ctx->panel);
+}
+
+static void exynos_dpi_dpms(struct exynos_drm_display *display, int mode)
+{
+        struct exynos_dpi *ctx = display->ctx;
+
+        switch (mode) {
+        case DRM_MODE_DPMS_ON:
+                if (ctx->dpms_mode != DRM_MODE_DPMS_ON)
+                                exynos_dpi_poweron(ctx);
+                        break;
+        case DRM_MODE_DPMS_STANDBY:
+        case DRM_MODE_DPMS_SUSPEND:
+        case DRM_MODE_DPMS_OFF:
+                if (ctx->dpms_mode == DRM_MODE_DPMS_ON)
+                        exynos_dpi_poweroff(ctx);
+                break;
+        default:
+                break;
+        };
+        ctx->dpms_mode = mode;
+}
+
+static struct exynos_drm_display_ops exynos_dpi_display_ops = {
+        .create_connector = exynos_dpi_create_connector,
+        .dpms = exynos_dpi_dpms
+};
+
+static struct exynos_drm_display exynos_dpi_display = {
+        .type = EXYNOS_DISPLAY_TYPE_LCD,
+        .ops = &exynos_dpi_display_ops,
+};
+
+/* of_* functions will be removed after merge of of_graph patches */
+static struct device_node *
+of_get_child_by_name_reg(struct device_node *parent, const char *name, u32 reg)
+{
+        struct device_node *np;
+
+        for_each_child_of_node(parent, np) {
+                u32 r;
+
+                if (!np->name || of_node_cmp(np->name, name))
+                        continue;
+
+                if (of_property_read_u32(np, "reg", &r) < 0)
+                        r = 0;
+
+                if (reg == r)
+                        break;
+        }
+
+        return np;
+}
+
+static struct device_node *of_graph_get_port_by_reg(struct device_node *parent,
+                                                    u32 reg)
+{
+        struct device_node *ports, *port;
+
+        ports = of_get_child_by_name(parent, "ports");
+        if (ports)
+                parent = ports;
+
+        port = of_get_child_by_name_reg(parent, "port", reg);
+
+        of_node_put(ports);
+
+        return port;
+}
+
+static struct device_node *
+of_graph_get_endpoint_by_reg(struct device_node *port, u32 reg)
+{
+        return of_get_child_by_name_reg(port, "endpoint", reg);
+}
+
+static struct device_node *
+of_graph_get_remote_port_parent(const struct device_node *node)
+{
+        struct device_node *np;
+        unsigned int depth;
+
+        np = of_parse_phandle(node, "remote-endpoint", 0);
+
+        /* Walk 3 levels up only if there is 'ports' node. */
+        for (depth = 3; depth && np; depth--) {
+                np = of_get_next_parent(np);
+                if (depth == 2 && of_node_cmp(np->name, "ports"))
+                        break;
+        }
+        return np;
+}
+
+enum {
+        FIMD_PORT_IN0,
+        FIMD_PORT_IN1,
+        FIMD_PORT_IN2,
+        FIMD_PORT_RGB,
+        FIMD_PORT_WRB,
+};
+
+static struct device_node *exynos_dpi_of_find_panel_node(struct device *dev)
+{
+        struct device_node *np, *ep;
+
+        np = of_graph_get_port_by_reg(dev->of_node, FIMD_PORT_RGB);
+        if (!np)
+                return NULL;
+
+        ep = of_graph_get_endpoint_by_reg(np, 0);
+        of_node_put(np);
+        if (!ep)
+                return NULL;
+
+        np = of_graph_get_remote_port_parent(ep);
+        of_node_put(ep);
+
+        return np;
+}
+
+static int exynos_dpi_parse_dt(struct exynos_dpi *ctx)
+{
+        struct device *dev = ctx->dev;
+        struct device_node *dn = dev->of_node;
+        struct device_node *np;
+
+        ctx->panel_node = exynos_dpi_of_find_panel_node(dev);
+
+        np = of_get_child_by_name(dn, "display-timings");
+        if (np) {
+                struct videomode *vm;
+                int ret;
+
+                of_node_put(np);
+
+                vm = devm_kzalloc(dev, sizeof(*ctx->vm), GFP_KERNEL);
+                if (!vm)
+                        return -ENOMEM;
+
+                ret = of_get_videomode(dn, vm, 0);
+                if (ret < 0)
+                        return ret;
+
+                ctx->vm = vm;
+
+                return 0;
+        }
+
+        if (!ctx->panel_node)
+                return -EINVAL;
+
+        return 0;
+}
+
+int exynos_dpi_probe(struct device *dev)
+{
+        struct exynos_dpi *ctx;
+        int ret;
+
+        ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
+        if (!ctx)
+                return -ENOMEM;
+
+        ctx->dev = dev;
+        exynos_dpi_display.ctx = ctx;
+        ctx->dpms_mode = DRM_MODE_DPMS_OFF;
+
+        ret = exynos_dpi_parse_dt(ctx);
+        if (ret < 0)
+                return ret;
+
+        exynos_drm_display_register(&exynos_dpi_display);
+
+        return 0;
+}
+
+int exynos_dpi_remove(struct device *dev)
+{
+        exynos_dpi_dpms(&exynos_dpi_display, DRM_MODE_DPMS_OFF);
+        exynos_drm_display_unregister(&exynos_dpi_display);
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.c b/drivers/gpu/drm/exynos/exynos_drm_drv.c
index c204b4e3356e..2d27ba23a6a8 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.c
@@ -11,6 +11,7 @@
  * option) any later version.
  */
 
+#include <linux/pm_runtime.h>
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
 
@@ -53,6 +54,7 @@ static int exynos_drm_load(struct drm_device *dev, unsigned long flags)
                 return -ENOMEM;
 
         INIT_LIST_HEAD(&private->pageflip_event_list);
+        dev_set_drvdata(dev->dev, dev);
         dev->dev_private = (void *)private;
 
         /*
@@ -64,38 +66,36 @@ static int exynos_drm_load(struct drm_device *dev, unsigned long flags)
         ret = drm_create_iommu_mapping(dev);
         if (ret < 0) {
                 DRM_ERROR("failed to create iommu mapping.\n");
-                goto err_crtc;
+                goto err_free_private;
         }
 
         drm_mode_config_init(dev);
 
-        /* init kms poll for handling hpd */
-        drm_kms_helper_poll_init(dev);
-
         exynos_drm_mode_config_init(dev);
 
-        /*
-         * EXYNOS4 is enough to have two CRTCs and each crtc would be used
-         * without dependency of hardware.
-         */
-        for (nr = 0; nr < MAX_CRTC; nr++) {
-                ret = exynos_drm_crtc_create(dev, nr);
-                if (ret)
-                        goto err_release_iommu_mapping;
-        }
+        ret = exynos_drm_initialize_managers(dev);
+        if (ret)
+                goto err_mode_config_cleanup;
 
         for (nr = 0; nr < MAX_PLANE; nr++) {
                 struct drm_plane *plane;
-                unsigned int possible_crtcs = (1 << MAX_CRTC) - 1;
+                unsigned long possible_crtcs = (1 << MAX_CRTC) - 1;
 
                 plane = exynos_plane_init(dev, possible_crtcs, false);
                 if (!plane)
-                        goto err_release_iommu_mapping;
+                        goto err_manager_cleanup;
         }
 
+        ret = exynos_drm_initialize_displays(dev);
+        if (ret)
+                goto err_manager_cleanup;
+
+        /* init kms poll for handling hpd */
+        drm_kms_helper_poll_init(dev);
+
         ret = drm_vblank_init(dev, MAX_CRTC);
         if (ret)
-                goto err_release_iommu_mapping;
+                goto err_display_cleanup;
 
         /*
          * probe sub drivers such as display controller and hdmi driver,
@@ -109,30 +109,25 @@ static int exynos_drm_load(struct drm_device *dev, unsigned long flags)
         /* setup possible_clones. */
         exynos_drm_encoder_setup(dev);
 
-        /*
-         * create and configure fb helper and also exynos specific
-         * fbdev object.
-         */
-        ret = exynos_drm_fbdev_init(dev);
-        if (ret) {
-                DRM_ERROR("failed to initialize drm fbdev\n");
-                goto err_drm_device;
-        }
-
         drm_vblank_offdelay = VBLANK_OFF_DELAY;
 
         platform_set_drvdata(dev->platformdev, dev);
 
+        /* force connectors detection */
+        drm_helper_hpd_irq_event(dev);
+
         return 0;
 
-err_drm_device:
-        exynos_drm_device_unregister(dev);
 err_vblank:
         drm_vblank_cleanup(dev);
-err_release_iommu_mapping:
-        drm_release_iommu_mapping(dev);
-err_crtc:
+err_display_cleanup:
+        exynos_drm_remove_displays(dev);
+err_manager_cleanup:
+        exynos_drm_remove_managers(dev);
+err_mode_config_cleanup:
         drm_mode_config_cleanup(dev);
+        drm_release_iommu_mapping(dev);
+err_free_private:
         kfree(private);
 
         return ret;
@@ -144,6 +139,8 @@ static int exynos_drm_unload(struct drm_device *dev)
         exynos_drm_device_unregister(dev);
         drm_vblank_cleanup(dev);
         drm_kms_helper_poll_fini(dev);
+        exynos_drm_remove_displays(dev);
+        exynos_drm_remove_managers(dev);
         drm_mode_config_cleanup(dev);
 
         drm_release_iommu_mapping(dev);
@@ -158,6 +155,41 @@ static const struct file_operations exynos_drm_gem_fops = {
         .mmap = exynos_drm_gem_mmap_buffer,
 };
 
+static int exynos_drm_suspend(struct drm_device *dev, pm_message_t state)
+{
+        struct drm_connector *connector;
+
+        drm_modeset_lock_all(dev);
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                int old_dpms = connector->dpms;
+
+                if (connector->funcs->dpms)
+                        connector->funcs->dpms(connector, DRM_MODE_DPMS_OFF);
+
+                /* Set the old mode back to the connector for resume */
+                connector->dpms = old_dpms;
+        }
+        drm_modeset_unlock_all(dev);
+
+        return 0;
+}
+
+static int exynos_drm_resume(struct drm_device *dev)
+{
+        struct drm_connector *connector;
+
+        drm_modeset_lock_all(dev);
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                if (connector->funcs->dpms)
+                        connector->funcs->dpms(connector, connector->dpms);
+        }
+
+        drm_helper_resume_force_mode(dev);
+        drm_modeset_unlock_all(dev);
+
+        return 0;
+}
+
 static int exynos_drm_open(struct drm_device *dev, struct drm_file *file)
 {
         struct drm_exynos_file_private *file_priv;
@@ -295,6 +327,8 @@ static struct drm_driver exynos_drm_driver = {
                                         DRIVER_GEM | DRIVER_PRIME,
         .load                   = exynos_drm_load,
         .unload                 = exynos_drm_unload,
+        .suspend                = exynos_drm_suspend,
+        .resume                 = exynos_drm_resume,
         .open                   = exynos_drm_open,
         .preclose               = exynos_drm_preclose,
         .lastclose              = exynos_drm_lastclose,
@@ -329,6 +363,9 @@ static int exynos_drm_platform_probe(struct platform_device *pdev)
         if (ret)
                 return ret;
 
+        pm_runtime_enable(&pdev->dev);
+        pm_runtime_get_sync(&pdev->dev);
+
         return drm_platform_init(&exynos_drm_driver, pdev);
 }
 
@@ -339,12 +376,67 @@ static int exynos_drm_platform_remove(struct platform_device *pdev)
         return 0;
 }
 
+#ifdef CONFIG_PM_SLEEP
+static int exynos_drm_sys_suspend(struct device *dev)
+{
+        struct drm_device *drm_dev = dev_get_drvdata(dev);
+        pm_message_t message;
+
+        if (pm_runtime_suspended(dev))
+                return 0;
+
+        message.event = PM_EVENT_SUSPEND;
+        return exynos_drm_suspend(drm_dev, message);
+}
+
+static int exynos_drm_sys_resume(struct device *dev)
+{
+        struct drm_device *drm_dev = dev_get_drvdata(dev);
+
+        if (pm_runtime_suspended(dev))
+                return 0;
+
+        return exynos_drm_resume(drm_dev);
+}
+#endif
+
+#ifdef CONFIG_PM_RUNTIME
+static int exynos_drm_runtime_suspend(struct device *dev)
+{
+        struct drm_device *drm_dev = dev_get_drvdata(dev);
+        pm_message_t message;
+
+        if (pm_runtime_suspended(dev))
+                return 0;
+
+        message.event = PM_EVENT_SUSPEND;
+        return exynos_drm_suspend(drm_dev, message);
+}
+
+static int exynos_drm_runtime_resume(struct device *dev)
+{
+        struct drm_device *drm_dev = dev_get_drvdata(dev);
+
+        if (!pm_runtime_suspended(dev))
+                return 0;
+
+        return exynos_drm_resume(drm_dev);
+}
+#endif
+
+static const struct dev_pm_ops exynos_drm_pm_ops = {
+        SET_SYSTEM_SLEEP_PM_OPS(exynos_drm_sys_suspend, exynos_drm_sys_resume)
+        SET_RUNTIME_PM_OPS(exynos_drm_runtime_suspend,
+                        exynos_drm_runtime_resume, NULL)
+};
+
 static struct platform_driver exynos_drm_platform_driver = {
         .probe          = exynos_drm_platform_probe,
         .remove         = exynos_drm_platform_remove,
         .driver         = {
                 .owner  = THIS_MODULE,
                 .name   = "exynos-drm",
+                .pm     = &exynos_drm_pm_ops,
         },
 };
 
@@ -352,6 +444,18 @@ static int __init exynos_drm_init(void)
 {
         int ret;
 
+#ifdef CONFIG_DRM_EXYNOS_DP
+        ret = platform_driver_register(&dp_driver);
+        if (ret < 0)
+                goto out_dp;
+#endif
+
+#ifdef CONFIG_DRM_EXYNOS_DSI
+        ret = platform_driver_register(&dsi_driver);
+        if (ret < 0)
+                goto out_dsi;
+#endif
+
 #ifdef CONFIG_DRM_EXYNOS_FIMD
         ret = platform_driver_register(&fimd_driver);
         if (ret < 0)
@@ -365,13 +469,6 @@ static int __init exynos_drm_init(void)
         ret = platform_driver_register(&mixer_driver);
         if (ret < 0)
                 goto out_mixer;
-        ret = platform_driver_register(&exynos_drm_common_hdmi_driver);
-        if (ret < 0)
-                goto out_common_hdmi;
-
-        ret = exynos_platform_device_hdmi_register();
-        if (ret < 0)
-                goto out_common_hdmi_dev;
 #endif
 
 #ifdef CONFIG_DRM_EXYNOS_VIDI
@@ -464,10 +561,6 @@ out_vidi:
 #endif
 
 #ifdef CONFIG_DRM_EXYNOS_HDMI
-        exynos_platform_device_hdmi_unregister();
-out_common_hdmi_dev:
-        platform_driver_unregister(&exynos_drm_common_hdmi_driver);
-out_common_hdmi:
         platform_driver_unregister(&mixer_driver);
 out_mixer:
         platform_driver_unregister(&hdmi_driver);
@@ -478,6 +571,16 @@ out_hdmi:
         platform_driver_unregister(&fimd_driver);
 out_fimd:
 #endif
+
+#ifdef CONFIG_DRM_EXYNOS_DSI
+        platform_driver_unregister(&dsi_driver);
+out_dsi:
+#endif
+
+#ifdef CONFIG_DRM_EXYNOS_DP
+        platform_driver_unregister(&dp_driver);
+out_dp:
+#endif
         return ret;
 }
 
@@ -509,8 +612,6 @@ static void __exit exynos_drm_exit(void)
 #endif
 
 #ifdef CONFIG_DRM_EXYNOS_HDMI
-        exynos_platform_device_hdmi_unregister();
-        platform_driver_unregister(&exynos_drm_common_hdmi_driver);
         platform_driver_unregister(&mixer_driver);
         platform_driver_unregister(&hdmi_driver);
 #endif
@@ -522,6 +623,14 @@ static void __exit exynos_drm_exit(void)
 #ifdef CONFIG_DRM_EXYNOS_FIMD
         platform_driver_unregister(&fimd_driver);
 #endif
+
+#ifdef CONFIG_DRM_EXYNOS_DSI
+        platform_driver_unregister(&dsi_driver);
+#endif
+
+#ifdef CONFIG_DRM_EXYNOS_DP
+        platform_driver_unregister(&dp_driver);
+#endif
 }
 
 module_init(exynos_drm_init);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h
index a8f9dba2a816..ce3e6a30deaa 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h
@@ -54,22 +54,6 @@ enum exynos_drm_output_type {
 };
 
 /*
- * Exynos drm overlay ops structure.
- *
- * @mode_set: copy drm overlay info to hw specific overlay info.
- * @commit: apply hardware specific overlay data to registers.
- * @enable: enable hardware specific overlay.
- * @disable: disable hardware specific overlay.
- */
-struct exynos_drm_overlay_ops {
-        void (*mode_set)(struct device *subdrv_dev,
-                         struct exynos_drm_overlay *overlay);
-        void (*commit)(struct device *subdrv_dev, int zpos);
-        void (*enable)(struct device *subdrv_dev, int zpos);
-        void (*disable)(struct device *subdrv_dev, int zpos);
-};
-
-/*
  * Exynos drm common overlay structure.
  *
  * @fb_x: offset x on a framebuffer to be displayed.
@@ -138,77 +122,110 @@ struct exynos_drm_overlay {
  * Exynos DRM Display Structure.
  *      - this structure is common to analog tv, digital tv and lcd panel.
  *
- * @type: one of EXYNOS_DISPLAY_TYPE_LCD and HDMI.
- * @is_connected: check for that display is connected or not.
- * @get_edid: get edid modes from display driver.
- * @get_panel: get panel object from display driver.
+ * @initialize: initializes the display with drm_dev
+ * @remove: cleans up the display for removal
+ * @mode_fixup: fix mode data comparing to hw specific display mode.
+ * @mode_set: convert drm_display_mode to hw specific display mode and
+ *            would be called by encoder->mode_set().
  * @check_mode: check if mode is valid or not.
- * @power_on: display device on or off.
+ * @dpms: display device on or off.
+ * @commit: apply changes to hw
  */
+struct exynos_drm_display;
 struct exynos_drm_display_ops {
+        int (*initialize)(struct exynos_drm_display *display,
+                                struct drm_device *drm_dev);
+        int (*create_connector)(struct exynos_drm_display *display,
+                                struct drm_encoder *encoder);
+        void (*remove)(struct exynos_drm_display *display);
+        void (*mode_fixup)(struct exynos_drm_display *display,
+                                struct drm_connector *connector,
+                                const struct drm_display_mode *mode,
+                                struct drm_display_mode *adjusted_mode);
+        void (*mode_set)(struct exynos_drm_display *display,
+                                struct drm_display_mode *mode);
+        int (*check_mode)(struct exynos_drm_display *display,
+                                struct drm_display_mode *mode);
+        void (*dpms)(struct exynos_drm_display *display, int mode);
+        void (*commit)(struct exynos_drm_display *display);
+};
+
+/*
+ * Exynos drm display structure, maps 1:1 with an encoder/connector
+ *
+ * @list: the list entry for this manager
+ * @type: one of EXYNOS_DISPLAY_TYPE_LCD and HDMI.
+ * @encoder: encoder object this display maps to
+ * @connector: connector object this display maps to
+ * @ops: pointer to callbacks for exynos drm specific functionality
+ * @ctx: A pointer to the display's implementation specific context
+ */
+struct exynos_drm_display {
+        struct list_head list;
         enum exynos_drm_output_type type;
-        bool (*is_connected)(struct device *dev);
-        struct edid *(*get_edid)(struct device *dev,
-                        struct drm_connector *connector);
-        void *(*get_panel)(struct device *dev);
-        int (*check_mode)(struct device *dev, struct drm_display_mode *mode);
-        int (*power_on)(struct device *dev, int mode);
+        struct drm_encoder *encoder;
+        struct drm_connector *connector;
+        struct exynos_drm_display_ops *ops;
+        void *ctx;
 };
 
 /*
  * Exynos drm manager ops
  *
+ * @initialize: initializes the manager with drm_dev
+ * @remove: cleans up the manager for removal
  * @dpms: control device power.
- * @apply: set timing, vblank and overlay data to registers.
- * @mode_fixup: fix mode data comparing to hw specific display mode.
- * @mode_set: convert drm_display_mode to hw specific display mode and
- *            would be called by encoder->mode_set().
- * @get_max_resol: get maximum resolution to specific hardware.
+ * @mode_fixup: fix mode data before applying it
+ * @mode_set: set the given mode to the manager
  * @commit: set current hw specific display mode to hw.
  * @enable_vblank: specific driver callback for enabling vblank interrupt.
  * @disable_vblank: specific driver callback for disabling vblank interrupt.
  * @wait_for_vblank: wait for vblank interrupt to make sure that
  *      hardware overlay is updated.
+ * @win_mode_set: copy drm overlay info to hw specific overlay info.
+ * @win_commit: apply hardware specific overlay data to registers.
+ * @win_enable: enable hardware specific overlay.
+ * @win_disable: disable hardware specific overlay.
  */
+struct exynos_drm_manager;
 struct exynos_drm_manager_ops {
-        void (*dpms)(struct device *subdrv_dev, int mode);
-        void (*apply)(struct device *subdrv_dev);
-        void (*mode_fixup)(struct device *subdrv_dev,
-                                struct drm_connector *connector,
+        int (*initialize)(struct exynos_drm_manager *mgr,
+                                struct drm_device *drm_dev, int pipe);
+        void (*remove)(struct exynos_drm_manager *mgr);
+        void (*dpms)(struct exynos_drm_manager *mgr, int mode);
+        bool (*mode_fixup)(struct exynos_drm_manager *mgr,
                                 const struct drm_display_mode *mode,
                                 struct drm_display_mode *adjusted_mode);
-        void (*mode_set)(struct device *subdrv_dev, void *mode);
-        void (*get_max_resol)(struct device *subdrv_dev, unsigned int *width,
-                                unsigned int *height);
-        void (*commit)(struct device *subdrv_dev);
-        int (*enable_vblank)(struct device *subdrv_dev);
-        void (*disable_vblank)(struct device *subdrv_dev);
-        void (*wait_for_vblank)(struct device *subdrv_dev);
+        void (*mode_set)(struct exynos_drm_manager *mgr,
+                                const struct drm_display_mode *mode);
+        void (*commit)(struct exynos_drm_manager *mgr);
+        int (*enable_vblank)(struct exynos_drm_manager *mgr);
+        void (*disable_vblank)(struct exynos_drm_manager *mgr);
+        void (*wait_for_vblank)(struct exynos_drm_manager *mgr);
+        void (*win_mode_set)(struct exynos_drm_manager *mgr,
+                                struct exynos_drm_overlay *overlay);
+        void (*win_commit)(struct exynos_drm_manager *mgr, int zpos);
+        void (*win_enable)(struct exynos_drm_manager *mgr, int zpos);
+        void (*win_disable)(struct exynos_drm_manager *mgr, int zpos);
 };
 
 /*
- * Exynos drm common manager structure.
+ * Exynos drm common manager structure, maps 1:1 with a crtc
  *
- * @dev: pointer to device object for subdrv device driver.
- *      sub drivers such as display controller or hdmi driver,
- *      have their own device object.
- * @ops: pointer to callbacks for exynos drm specific framebuffer.
- *      these callbacks should be set by specific drivers such fimd
- *      or hdmi driver and are used to control hardware global registers.
- * @overlay_ops: pointer to callbacks for exynos drm specific framebuffer.
- *      these callbacks should be set by specific drivers such fimd
- *      or hdmi driver and are used to control hardware overlay reigsters.
- * @display: pointer to callbacks for exynos drm specific framebuffer.
- *      these callbacks should be set by specific drivers such fimd
- *      or hdmi driver and are used to control display devices such as
- *      analog tv, digital tv and lcd panel and also get timing data for them.
+ * @list: the list entry for this manager
+ * @type: one of EXYNOS_DISPLAY_TYPE_LCD and HDMI.
+ * @drm_dev: pointer to the drm device
+ * @pipe: the pipe number for this crtc/manager
+ * @ops: pointer to callbacks for exynos drm specific functionality
+ * @ctx: A pointer to the manager's implementation specific context
  */
 struct exynos_drm_manager {
-        struct device *dev;
+        struct list_head list;
+        enum exynos_drm_output_type type;
+        struct drm_device *drm_dev;
         int pipe;
         struct exynos_drm_manager_ops *ops;
-        struct exynos_drm_overlay_ops *overlay_ops;
-        struct exynos_drm_display_ops *display_ops;
+        void *ctx;
 };
 
 struct exynos_drm_g2d_private {
@@ -271,14 +288,11 @@ struct exynos_drm_private {
  *      by probe callback.
  * @open: this would be called with drm device file open.
  * @close: this would be called with drm device file close.
- * @encoder: encoder object owned by this sub driver.
- * @connector: connector object owned by this sub driver.
  */
 struct exynos_drm_subdrv {
         struct list_head list;
         struct device *dev;
         struct drm_device *drm_dev;
-        struct exynos_drm_manager *manager;
 
         int (*probe)(struct drm_device *drm_dev, struct device *dev);
         void (*remove)(struct drm_device *drm_dev, struct device *dev);
@@ -286,9 +300,6 @@ struct exynos_drm_subdrv {
                         struct drm_file *file);
         void (*close)(struct drm_device *drm_dev, struct device *dev,
                         struct drm_file *file);
-
-        struct drm_encoder *encoder;
-        struct drm_connector *connector;
 };
 
 /*
@@ -303,6 +314,16 @@ int exynos_drm_device_register(struct drm_device *dev);
  */
 int exynos_drm_device_unregister(struct drm_device *dev);
 
+int exynos_drm_initialize_managers(struct drm_device *dev);
+void exynos_drm_remove_managers(struct drm_device *dev);
+int exynos_drm_initialize_displays(struct drm_device *dev);
+void exynos_drm_remove_displays(struct drm_device *dev);
+
+int exynos_drm_manager_register(struct exynos_drm_manager *manager);
+int exynos_drm_manager_unregister(struct exynos_drm_manager *manager);
+int exynos_drm_display_register(struct exynos_drm_display *display);
+int exynos_drm_display_unregister(struct exynos_drm_display *display);
+
 /*
  * this function would be called by sub drivers such as display controller
  * or hdmi driver to register this sub driver object to exynos drm driver
@@ -338,6 +359,16 @@ int exynos_platform_device_ipp_register(void);
  */
 void exynos_platform_device_ipp_unregister(void);
 
+#ifdef CONFIG_DRM_EXYNOS_DPI
+int exynos_dpi_probe(struct device *dev);
+int exynos_dpi_remove(struct device *dev);
+#else
+static inline int exynos_dpi_probe(struct device *dev) { return 0; }
+static inline int exynos_dpi_remove(struct device *dev) { return 0; }
+#endif
+
+extern struct platform_driver dp_driver;
+extern struct platform_driver dsi_driver;
 extern struct platform_driver fimd_driver;
 extern struct platform_driver hdmi_driver;
 extern struct platform_driver mixer_driver;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dsi.c b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
new file mode 100644
index 000000000000..eb73e3bf2a0c
--- /dev/null
+++ b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
@@ -0,0 +1,1524 @@
+/*
+ * Samsung SoC MIPI DSI Master driver.
+ *
+ * Copyright (c) 2014 Samsung Electronics Co., Ltd
+ *
+ * Contacts: Tomasz Figa <t.figa@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+*/
+
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_panel.h>
+
+#include <linux/clk.h>
+#include <linux/irq.h>
+#include <linux/phy/phy.h>
+#include <linux/regulator/consumer.h>
+
+#include <video/mipi_display.h>
+#include <video/videomode.h>
+
+#include "exynos_drm_drv.h"
+
+/* returns true iff both arguments logically differs */
+#define NEQV(a, b) (!(a) ^ !(b))
+
+#define DSIM_STATUS_REG         0x0     /* Status register */
+#define DSIM_SWRST_REG          0x4     /* Software reset register */
+#define DSIM_CLKCTRL_REG        0x8     /* Clock control register */
+#define DSIM_TIMEOUT_REG        0xc     /* Time out register */
+#define DSIM_CONFIG_REG         0x10    /* Configuration register */
+#define DSIM_ESCMODE_REG        0x14    /* Escape mode register */
+
+/* Main display image resolution register */
+#define DSIM_MDRESOL_REG        0x18
+#define DSIM_MVPORCH_REG        0x1c    /* Main display Vporch register */
+#define DSIM_MHPORCH_REG        0x20    /* Main display Hporch register */
+#define DSIM_MSYNC_REG          0x24    /* Main display sync area register */
+
+/* Sub display image resolution register */
+#define DSIM_SDRESOL_REG        0x28
+#define DSIM_INTSRC_REG         0x2c    /* Interrupt source register */
+#define DSIM_INTMSK_REG         0x30    /* Interrupt mask register */
+#define DSIM_PKTHDR_REG         0x34    /* Packet Header FIFO register */
+#define DSIM_PAYLOAD_REG        0x38    /* Payload FIFO register */
+#define DSIM_RXFIFO_REG         0x3c    /* Read FIFO register */
+#define DSIM_FIFOTHLD_REG       0x40    /* FIFO threshold level register */
+#define DSIM_FIFOCTRL_REG       0x44    /* FIFO status and control register */
+
+/* FIFO memory AC characteristic register */
+#define DSIM_PLLCTRL_REG        0x4c    /* PLL control register */
+#define DSIM_PLLTMR_REG         0x50    /* PLL timer register */
+#define DSIM_PHYACCHR_REG       0x54    /* D-PHY AC characteristic register */
+#define DSIM_PHYACCHR1_REG      0x58    /* D-PHY AC characteristic register1 */
+
+/* DSIM_STATUS */
+#define DSIM_STOP_STATE_DAT(x)          (((x) & 0xf) << 0)
+#define DSIM_STOP_STATE_CLK             (1 << 8)
+#define DSIM_TX_READY_HS_CLK            (1 << 10)
+#define DSIM_PLL_STABLE                 (1 << 31)
+
+/* DSIM_SWRST */
+#define DSIM_FUNCRST                    (1 << 16)
+#define DSIM_SWRST                      (1 << 0)
+
+/* DSIM_TIMEOUT */
+#define DSIM_LPDR_TIMEOUT(x)            ((x) << 0)
+#define DSIM_BTA_TIMEOUT(x)             ((x) << 16)
+
+/* DSIM_CLKCTRL */
+#define DSIM_ESC_PRESCALER(x)           (((x) & 0xffff) << 0)
+#define DSIM_ESC_PRESCALER_MASK         (0xffff << 0)
+#define DSIM_LANE_ESC_CLK_EN_CLK        (1 << 19)
+#define DSIM_LANE_ESC_CLK_EN_DATA(x)    (((x) & 0xf) << 20)
+#define DSIM_LANE_ESC_CLK_EN_DATA_MASK  (0xf << 20)
+#define DSIM_BYTE_CLKEN                 (1 << 24)
+#define DSIM_BYTE_CLK_SRC(x)            (((x) & 0x3) << 25)
+#define DSIM_BYTE_CLK_SRC_MASK          (0x3 << 25)
+#define DSIM_PLL_BYPASS                 (1 << 27)
+#define DSIM_ESC_CLKEN                  (1 << 28)
+#define DSIM_TX_REQUEST_HSCLK           (1 << 31)
+
+/* DSIM_CONFIG */
+#define DSIM_LANE_EN_CLK                (1 << 0)
+#define DSIM_LANE_EN(x)                 (((x) & 0xf) << 1)
+#define DSIM_NUM_OF_DATA_LANE(x)        (((x) & 0x3) << 5)
+#define DSIM_SUB_PIX_FORMAT(x)          (((x) & 0x7) << 8)
+#define DSIM_MAIN_PIX_FORMAT_MASK       (0x7 << 12)
+#define DSIM_MAIN_PIX_FORMAT_RGB888     (0x7 << 12)
+#define DSIM_MAIN_PIX_FORMAT_RGB666     (0x6 << 12)
+#define DSIM_MAIN_PIX_FORMAT_RGB666_P   (0x5 << 12)
+#define DSIM_MAIN_PIX_FORMAT_RGB565     (0x4 << 12)
+#define DSIM_SUB_VC                     (((x) & 0x3) << 16)
+#define DSIM_MAIN_VC                    (((x) & 0x3) << 18)
+#define DSIM_HSA_MODE                   (1 << 20)
+#define DSIM_HBP_MODE                   (1 << 21)
+#define DSIM_HFP_MODE                   (1 << 22)
+#define DSIM_HSE_MODE                   (1 << 23)
+#define DSIM_AUTO_MODE                  (1 << 24)
+#define DSIM_VIDEO_MODE                 (1 << 25)
+#define DSIM_BURST_MODE                 (1 << 26)
+#define DSIM_SYNC_INFORM                (1 << 27)
+#define DSIM_EOT_DISABLE                (1 << 28)
+#define DSIM_MFLUSH_VS                  (1 << 29)
+
+/* DSIM_ESCMODE */
+#define DSIM_TX_TRIGGER_RST             (1 << 4)
+#define DSIM_TX_LPDT_LP                 (1 << 6)
+#define DSIM_CMD_LPDT_LP                (1 << 7)
+#define DSIM_FORCE_BTA                  (1 << 16)
+#define DSIM_FORCE_STOP_STATE           (1 << 20)
+#define DSIM_STOP_STATE_CNT(x)          (((x) & 0x7ff) << 21)
+#define DSIM_STOP_STATE_CNT_MASK        (0x7ff << 21)
+
+/* DSIM_MDRESOL */
+#define DSIM_MAIN_STAND_BY              (1 << 31)
+#define DSIM_MAIN_VRESOL(x)             (((x) & 0x7ff) << 16)
+#define DSIM_MAIN_HRESOL(x)             (((x) & 0X7ff) << 0)
+
+/* DSIM_MVPORCH */
+#define DSIM_CMD_ALLOW(x)               ((x) << 28)
+#define DSIM_STABLE_VFP(x)              ((x) << 16)
+#define DSIM_MAIN_VBP(x)                ((x) << 0)
+#define DSIM_CMD_ALLOW_MASK             (0xf << 28)
+#define DSIM_STABLE_VFP_MASK            (0x7ff << 16)
+#define DSIM_MAIN_VBP_MASK              (0x7ff << 0)
+
+/* DSIM_MHPORCH */
+#define DSIM_MAIN_HFP(x)                ((x) << 16)
+#define DSIM_MAIN_HBP(x)                ((x) << 0)
+#define DSIM_MAIN_HFP_MASK              ((0xffff) << 16)
+#define DSIM_MAIN_HBP_MASK              ((0xffff) << 0)
+
+/* DSIM_MSYNC */
+#define DSIM_MAIN_VSA(x)                ((x) << 22)
+#define DSIM_MAIN_HSA(x)                ((x) << 0)
+#define DSIM_MAIN_VSA_MASK              ((0x3ff) << 22)
+#define DSIM_MAIN_HSA_MASK              ((0xffff) << 0)
+
+/* DSIM_SDRESOL */
+#define DSIM_SUB_STANDY(x)              ((x) << 31)
+#define DSIM_SUB_VRESOL(x)              ((x) << 16)
+#define DSIM_SUB_HRESOL(x)              ((x) << 0)
+#define DSIM_SUB_STANDY_MASK            ((0x1) << 31)
+#define DSIM_SUB_VRESOL_MASK            ((0x7ff) << 16)
+#define DSIM_SUB_HRESOL_MASK            ((0x7ff) << 0)
+
+/* DSIM_INTSRC */
+#define DSIM_INT_PLL_STABLE             (1 << 31)
+#define DSIM_INT_SW_RST_RELEASE         (1 << 30)
+#define DSIM_INT_SFR_FIFO_EMPTY         (1 << 29)
+#define DSIM_INT_BTA                    (1 << 25)
+#define DSIM_INT_FRAME_DONE             (1 << 24)
+#define DSIM_INT_RX_TIMEOUT             (1 << 21)
+#define DSIM_INT_BTA_TIMEOUT            (1 << 20)
+#define DSIM_INT_RX_DONE                (1 << 18)
+#define DSIM_INT_RX_TE                  (1 << 17)
+#define DSIM_INT_RX_ACK                 (1 << 16)
+#define DSIM_INT_RX_ECC_ERR             (1 << 15)
+#define DSIM_INT_RX_CRC_ERR             (1 << 14)
+
+/* DSIM_FIFOCTRL */
+#define DSIM_RX_DATA_FULL               (1 << 25)
+#define DSIM_RX_DATA_EMPTY              (1 << 24)
+#define DSIM_SFR_HEADER_FULL            (1 << 23)
+#define DSIM_SFR_HEADER_EMPTY           (1 << 22)
+#define DSIM_SFR_PAYLOAD_FULL           (1 << 21)
+#define DSIM_SFR_PAYLOAD_EMPTY          (1 << 20)
+#define DSIM_I80_HEADER_FULL            (1 << 19)
+#define DSIM_I80_HEADER_EMPTY           (1 << 18)
+#define DSIM_I80_PAYLOAD_FULL           (1 << 17)
+#define DSIM_I80_PAYLOAD_EMPTY          (1 << 16)
+#define DSIM_SD_HEADER_FULL             (1 << 15)
+#define DSIM_SD_HEADER_EMPTY            (1 << 14)
+#define DSIM_SD_PAYLOAD_FULL            (1 << 13)
+#define DSIM_SD_PAYLOAD_EMPTY           (1 << 12)
+#define DSIM_MD_HEADER_FULL             (1 << 11)
+#define DSIM_MD_HEADER_EMPTY            (1 << 10)
+#define DSIM_MD_PAYLOAD_FULL            (1 << 9)
+#define DSIM_MD_PAYLOAD_EMPTY           (1 << 8)
+#define DSIM_RX_FIFO                    (1 << 4)
+#define DSIM_SFR_FIFO                   (1 << 3)
+#define DSIM_I80_FIFO                   (1 << 2)
+#define DSIM_SD_FIFO                    (1 << 1)
+#define DSIM_MD_FIFO                    (1 << 0)
+
+/* DSIM_PHYACCHR */
+#define DSIM_AFC_EN                     (1 << 14)
+#define DSIM_AFC_CTL(x)                 (((x) & 0x7) << 5)
+
+/* DSIM_PLLCTRL */
+#define DSIM_FREQ_BAND(x)               ((x) << 24)
+#define DSIM_PLL_EN                     (1 << 23)
+#define DSIM_PLL_P(x)                   ((x) << 13)
+#define DSIM_PLL_M(x)                   ((x) << 4)
+#define DSIM_PLL_S(x)                   ((x) << 1)
+
+#define DSI_MAX_BUS_WIDTH               4
+#define DSI_NUM_VIRTUAL_CHANNELS        4
+#define DSI_TX_FIFO_SIZE                2048
+#define DSI_RX_FIFO_SIZE                256
+#define DSI_XFER_TIMEOUT_MS             100
+#define DSI_RX_FIFO_EMPTY               0x30800002
+
+enum exynos_dsi_transfer_type {
+        EXYNOS_DSI_TX,
+        EXYNOS_DSI_RX,
+};
+
+struct exynos_dsi_transfer {
+        struct list_head list;
+        struct completion completed;
+        int result;
+        u8 data_id;
+        u8 data[2];
+        u16 flags;
+
+        const u8 *tx_payload;
+        u16 tx_len;
+        u16 tx_done;
+
+        u8 *rx_payload;
+        u16 rx_len;
+        u16 rx_done;
+};
+
+#define DSIM_STATE_ENABLED              BIT(0)
+#define DSIM_STATE_INITIALIZED          BIT(1)
+#define DSIM_STATE_CMD_LPM              BIT(2)
+
+struct exynos_dsi {
+        struct mipi_dsi_host dsi_host;
+        struct drm_connector connector;
+        struct drm_encoder *encoder;
+        struct device_node *panel_node;
+        struct drm_panel *panel;
+        struct device *dev;
+
+        void __iomem *reg_base;
+        struct phy *phy;
+        struct clk *pll_clk;
+        struct clk *bus_clk;
+        struct regulator_bulk_data supplies[2];
+        int irq;
+
+        u32 pll_clk_rate;
+        u32 burst_clk_rate;
+        u32 esc_clk_rate;
+        u32 lanes;
+        u32 mode_flags;
+        u32 format;
+        struct videomode vm;
+
+        int state;
+        struct drm_property *brightness;
+        struct completion completed;
+
+        spinlock_t transfer_lock; /* protects transfer_list */
+        struct list_head transfer_list;
+};
+
+#define host_to_dsi(host) container_of(host, struct exynos_dsi, dsi_host)
+#define connector_to_dsi(c) container_of(c, struct exynos_dsi, connector)
+
+static void exynos_dsi_wait_for_reset(struct exynos_dsi *dsi)
+{
+        if (wait_for_completion_timeout(&dsi->completed, msecs_to_jiffies(300)))
+                return;
+
+        dev_err(dsi->dev, "timeout waiting for reset\n");
+}
+
+static void exynos_dsi_reset(struct exynos_dsi *dsi)
+{
+        reinit_completion(&dsi->completed);
+        writel(DSIM_SWRST, dsi->reg_base + DSIM_SWRST_REG);
+}
+
+#ifndef MHZ
+#define MHZ     (1000*1000)
+#endif
+
+static unsigned long exynos_dsi_pll_find_pms(struct exynos_dsi *dsi,
+                unsigned long fin, unsigned long fout, u8 *p, u16 *m, u8 *s)
+{
+        unsigned long best_freq = 0;
+        u32 min_delta = 0xffffffff;
+        u8 p_min, p_max;
+        u8 _p, uninitialized_var(best_p);
+        u16 _m, uninitialized_var(best_m);
+        u8 _s, uninitialized_var(best_s);
+
+        p_min = DIV_ROUND_UP(fin, (12 * MHZ));
+        p_max = fin / (6 * MHZ);
+
+        for (_p = p_min; _p <= p_max; ++_p) {
+                for (_s = 0; _s <= 5; ++_s) {
+                        u64 tmp;
+                        u32 delta;
+
+                        tmp = (u64)fout * (_p << _s);
+                        do_div(tmp, fin);
+                        _m = tmp;
+                        if (_m < 41 || _m > 125)
+                                continue;
+
+                        tmp = (u64)_m * fin;
+                        do_div(tmp, _p);
+                        if (tmp < 500 * MHZ || tmp > 1000 * MHZ)
+                                continue;
+
+                        tmp = (u64)_m * fin;
+                        do_div(tmp, _p << _s);
+
+                        delta = abs(fout - tmp);
+                        if (delta < min_delta) {
+                                best_p = _p;
+                                best_m = _m;
+                                best_s = _s;
+                                min_delta = delta;
+                                best_freq = tmp;
+                        }
+                }
+        }
+
+        if (best_freq) {
+                *p = best_p;
+                *m = best_m;
+                *s = best_s;
+        }
+
+        return best_freq;
+}
+
+static unsigned long exynos_dsi_set_pll(struct exynos_dsi *dsi,
+                                        unsigned long freq)
+{
+        static const unsigned long freq_bands[] = {
+                100 * MHZ, 120 * MHZ, 160 * MHZ, 200 * MHZ,
+                270 * MHZ, 320 * MHZ, 390 * MHZ, 450 * MHZ,
+                510 * MHZ, 560 * MHZ, 640 * MHZ, 690 * MHZ,
+                770 * MHZ, 870 * MHZ, 950 * MHZ,
+        };
+        unsigned long fin, fout;
+        int timeout, band;
+        u8 p, s;
+        u16 m;
+        u32 reg;
+
+        clk_set_rate(dsi->pll_clk, dsi->pll_clk_rate);
+
+        fin = clk_get_rate(dsi->pll_clk);
+        if (!fin) {
+                dev_err(dsi->dev, "failed to get PLL clock frequency\n");
+                return 0;
+        }
+
+        dev_dbg(dsi->dev, "PLL input frequency: %lu\n", fin);
+
+        fout = exynos_dsi_pll_find_pms(dsi, fin, freq, &p, &m, &s);
+        if (!fout) {
+                dev_err(dsi->dev,
+                        "failed to find PLL PMS for requested frequency\n");
+                return -EFAULT;
+        }
+
+        for (band = 0; band < ARRAY_SIZE(freq_bands); ++band)
+                if (fout < freq_bands[band])
+                        break;
+
+        dev_dbg(dsi->dev, "PLL freq %lu, (p %d, m %d, s %d), band %d\n", fout,
+                p, m, s, band);
+
+        writel(500, dsi->reg_base + DSIM_PLLTMR_REG);
+
+        reg = DSIM_FREQ_BAND(band) | DSIM_PLL_EN
+                        | DSIM_PLL_P(p) | DSIM_PLL_M(m) | DSIM_PLL_S(s);
+        writel(reg, dsi->reg_base + DSIM_PLLCTRL_REG);
+
+        timeout = 1000;
+        do {
+                if (timeout-- == 0) {
+                        dev_err(dsi->dev, "PLL failed to stabilize\n");
+                        return -EFAULT;
+                }
+                reg = readl(dsi->reg_base + DSIM_STATUS_REG);
+        } while ((reg & DSIM_PLL_STABLE) == 0);
+
+        return fout;
+}
+
+static int exynos_dsi_enable_clock(struct exynos_dsi *dsi)
+{
+        unsigned long hs_clk, byte_clk, esc_clk;
+        unsigned long esc_div;
+        u32 reg;
+
+        hs_clk = exynos_dsi_set_pll(dsi, dsi->burst_clk_rate);
+        if (!hs_clk) {
+                dev_err(dsi->dev, "failed to configure DSI PLL\n");
+                return -EFAULT;
+        }
+
+        byte_clk = hs_clk / 8;
+        esc_div = DIV_ROUND_UP(byte_clk, dsi->esc_clk_rate);
+        esc_clk = byte_clk / esc_div;
+
+        if (esc_clk > 20 * MHZ) {
+                ++esc_div;
+                esc_clk = byte_clk / esc_div;
+        }
+
+        dev_dbg(dsi->dev, "hs_clk = %lu, byte_clk = %lu, esc_clk = %lu\n",
+                hs_clk, byte_clk, esc_clk);
+
+        reg = readl(dsi->reg_base + DSIM_CLKCTRL_REG);
+        reg &= ~(DSIM_ESC_PRESCALER_MASK | DSIM_LANE_ESC_CLK_EN_CLK
+                        | DSIM_LANE_ESC_CLK_EN_DATA_MASK | DSIM_PLL_BYPASS
+                        | DSIM_BYTE_CLK_SRC_MASK);
+        reg |= DSIM_ESC_CLKEN | DSIM_BYTE_CLKEN
+                        | DSIM_ESC_PRESCALER(esc_div)
+                        | DSIM_LANE_ESC_CLK_EN_CLK
+                        | DSIM_LANE_ESC_CLK_EN_DATA(BIT(dsi->lanes) - 1)
+                        | DSIM_BYTE_CLK_SRC(0)
+                        | DSIM_TX_REQUEST_HSCLK;
+        writel(reg, dsi->reg_base + DSIM_CLKCTRL_REG);
+
+        return 0;
+}
+
+static void exynos_dsi_disable_clock(struct exynos_dsi *dsi)
+{
+        u32 reg;
+
+        reg = readl(dsi->reg_base + DSIM_CLKCTRL_REG);
+        reg &= ~(DSIM_LANE_ESC_CLK_EN_CLK | DSIM_LANE_ESC_CLK_EN_DATA_MASK
+                        | DSIM_ESC_CLKEN | DSIM_BYTE_CLKEN);
+        writel(reg, dsi->reg_base + DSIM_CLKCTRL_REG);
+
+        reg = readl(dsi->reg_base + DSIM_PLLCTRL_REG);
+        reg &= ~DSIM_PLL_EN;
+        writel(reg, dsi->reg_base + DSIM_PLLCTRL_REG);
+}
+
+static int exynos_dsi_init_link(struct exynos_dsi *dsi)
+{
+        int timeout;
+        u32 reg;
+        u32 lanes_mask;
+
+        /* Initialize FIFO pointers */
+        reg = readl(dsi->reg_base + DSIM_FIFOCTRL_REG);
+        reg &= ~0x1f;
+        writel(reg, dsi->reg_base + DSIM_FIFOCTRL_REG);
+
+        usleep_range(9000, 11000);
+
+        reg |= 0x1f;
+        writel(reg, dsi->reg_base + DSIM_FIFOCTRL_REG);
+
+        usleep_range(9000, 11000);
+
+        /* DSI configuration */
+        reg = 0;
+
+        if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO) {
+                reg |= DSIM_VIDEO_MODE;
+
+                if (!(dsi->mode_flags & MIPI_DSI_MODE_VSYNC_FLUSH))
+                        reg |= DSIM_MFLUSH_VS;
+                if (!(dsi->mode_flags & MIPI_DSI_MODE_EOT_PACKET))
+                        reg |= DSIM_EOT_DISABLE;
+                if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_SYNC_PULSE)
+                        reg |= DSIM_SYNC_INFORM;
+                if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_BURST)
+                        reg |= DSIM_BURST_MODE;
+                if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_AUTO_VERT)
+                        reg |= DSIM_AUTO_MODE;
+                if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO_HSE)
+                        reg |= DSIM_HSE_MODE;
+                if (!(dsi->mode_flags & MIPI_DSI_MODE_VIDEO_HFP))
+                        reg |= DSIM_HFP_MODE;
+                if (!(dsi->mode_flags & MIPI_DSI_MODE_VIDEO_HBP))
+                        reg |= DSIM_HBP_MODE;
+                if (!(dsi->mode_flags & MIPI_DSI_MODE_VIDEO_HSA))
+                        reg |= DSIM_HSA_MODE;
+        }
+
+        switch (dsi->format) {
+        case MIPI_DSI_FMT_RGB888:
+                reg |= DSIM_MAIN_PIX_FORMAT_RGB888;
+                break;
+        case MIPI_DSI_FMT_RGB666:
+                reg |= DSIM_MAIN_PIX_FORMAT_RGB666;
+                break;
+        case MIPI_DSI_FMT_RGB666_PACKED:
+                reg |= DSIM_MAIN_PIX_FORMAT_RGB666_P;
+                break;
+        case MIPI_DSI_FMT_RGB565:
+                reg |= DSIM_MAIN_PIX_FORMAT_RGB565;
+                break;
+        default:
+                dev_err(dsi->dev, "invalid pixel format\n");
+                return -EINVAL;
+        }
+
+        reg |= DSIM_NUM_OF_DATA_LANE(dsi->lanes - 1);
+
+        writel(reg, dsi->reg_base + DSIM_CONFIG_REG);
+
+        reg |= DSIM_LANE_EN_CLK;
+        writel(reg, dsi->reg_base + DSIM_CONFIG_REG);
+
+        lanes_mask = BIT(dsi->lanes) - 1;
+        reg |= DSIM_LANE_EN(lanes_mask);
+        writel(reg, dsi->reg_base + DSIM_CONFIG_REG);
+
+        /* Check clock and data lane state are stop state */
+        timeout = 100;
+        do {
+                if (timeout-- == 0) {
+                        dev_err(dsi->dev, "waiting for bus lanes timed out\n");
+                        return -EFAULT;
+                }
+
+                reg = readl(dsi->reg_base + DSIM_STATUS_REG);
+                if ((reg & DSIM_STOP_STATE_DAT(lanes_mask))
+                    != DSIM_STOP_STATE_DAT(lanes_mask))
+                        continue;
+        } while (!(reg & (DSIM_STOP_STATE_CLK | DSIM_TX_READY_HS_CLK)));
+
+        reg = readl(dsi->reg_base + DSIM_ESCMODE_REG);
+        reg &= ~DSIM_STOP_STATE_CNT_MASK;
+        reg |= DSIM_STOP_STATE_CNT(0xf);
+        writel(reg, dsi->reg_base + DSIM_ESCMODE_REG);
+
+        reg = DSIM_BTA_TIMEOUT(0xff) | DSIM_LPDR_TIMEOUT(0xffff);
+        writel(reg, dsi->reg_base + DSIM_TIMEOUT_REG);
+
+        return 0;
+}
+
+static void exynos_dsi_set_display_mode(struct exynos_dsi *dsi)
+{
+        struct videomode *vm = &dsi->vm;
+        u32 reg;
+
+        if (dsi->mode_flags & MIPI_DSI_MODE_VIDEO) {
+                reg = DSIM_CMD_ALLOW(0xf)
+                        | DSIM_STABLE_VFP(vm->vfront_porch)
+                        | DSIM_MAIN_VBP(vm->vback_porch);
+                writel(reg, dsi->reg_base + DSIM_MVPORCH_REG);
+
+                reg = DSIM_MAIN_HFP(vm->hfront_porch)
+                        | DSIM_MAIN_HBP(vm->hback_porch);
+                writel(reg, dsi->reg_base + DSIM_MHPORCH_REG);
+
+                reg = DSIM_MAIN_VSA(vm->vsync_len)
+                        | DSIM_MAIN_HSA(vm->hsync_len);
+                writel(reg, dsi->reg_base + DSIM_MSYNC_REG);
+        }
+
+        reg = DSIM_MAIN_HRESOL(vm->hactive) | DSIM_MAIN_VRESOL(vm->vactive);
+        writel(reg, dsi->reg_base + DSIM_MDRESOL_REG);
+
+        dev_dbg(dsi->dev, "LCD size = %dx%d\n", vm->hactive, vm->vactive);
+}
+
+static void exynos_dsi_set_display_enable(struct exynos_dsi *dsi, bool enable)
+{
+        u32 reg;
+
+        reg = readl(dsi->reg_base + DSIM_MDRESOL_REG);
+        if (enable)
+                reg |= DSIM_MAIN_STAND_BY;
+        else
+                reg &= ~DSIM_MAIN_STAND_BY;
+        writel(reg, dsi->reg_base + DSIM_MDRESOL_REG);
+}
+
+static int exynos_dsi_wait_for_hdr_fifo(struct exynos_dsi *dsi)
+{
+        int timeout = 2000;
+
+        do {
+                u32 reg = readl(dsi->reg_base + DSIM_FIFOCTRL_REG);
+
+                if (!(reg & DSIM_SFR_HEADER_FULL))
+                        return 0;
+
+                if (!cond_resched())
+                        usleep_range(950, 1050);
+        } while (--timeout);
+
+        return -ETIMEDOUT;
+}
+
+static void exynos_dsi_set_cmd_lpm(struct exynos_dsi *dsi, bool lpm)
+{
+        u32 v = readl(dsi->reg_base + DSIM_ESCMODE_REG);
+
+        if (lpm)
+                v |= DSIM_CMD_LPDT_LP;
+        else
+                v &= ~DSIM_CMD_LPDT_LP;
+
+        writel(v, dsi->reg_base + DSIM_ESCMODE_REG);
+}
+
+static void exynos_dsi_force_bta(struct exynos_dsi *dsi)
+{
+        u32 v = readl(dsi->reg_base + DSIM_ESCMODE_REG);
+
+        v |= DSIM_FORCE_BTA;
+        writel(v, dsi->reg_base + DSIM_ESCMODE_REG);
+}
+
+static void exynos_dsi_send_to_fifo(struct exynos_dsi *dsi,
+                                        struct exynos_dsi_transfer *xfer)
+{
+        struct device *dev = dsi->dev;
+        const u8 *payload = xfer->tx_payload + xfer->tx_done;
+        u16 length = xfer->tx_len - xfer->tx_done;
+        bool first = !xfer->tx_done;
+        u32 reg;
+
+        dev_dbg(dev, "< xfer %p: tx len %u, done %u, rx len %u, done %u\n",
+                xfer, xfer->tx_len, xfer->tx_done, xfer->rx_len, xfer->rx_done);
+
+        if (length > DSI_TX_FIFO_SIZE)
+                length = DSI_TX_FIFO_SIZE;
+
+        xfer->tx_done += length;
+
+        /* Send payload */
+        while (length >= 4) {
+                reg = (payload[3] << 24) | (payload[2] << 16)
+                                        | (payload[1] << 8) | payload[0];
+                writel(reg, dsi->reg_base + DSIM_PAYLOAD_REG);
+                payload += 4;
+                length -= 4;
+        }
+
+        reg = 0;
+        switch (length) {
+        case 3:
+                reg |= payload[2] << 16;
+                /* Fall through */
+        case 2:
+                reg |= payload[1] << 8;
+                /* Fall through */
+        case 1:
+                reg |= payload[0];
+                writel(reg, dsi->reg_base + DSIM_PAYLOAD_REG);
+                break;
+        case 0:
+                /* Do nothing */
+                break;
+        }
+
+        /* Send packet header */
+        if (!first)
+                return;
+
+        reg = (xfer->data[1] << 16) | (xfer->data[0] << 8) | xfer->data_id;
+        if (exynos_dsi_wait_for_hdr_fifo(dsi)) {
+                dev_err(dev, "waiting for header FIFO timed out\n");
+                return;
+        }
+
+        if (NEQV(xfer->flags & MIPI_DSI_MSG_USE_LPM,
+                 dsi->state & DSIM_STATE_CMD_LPM)) {
+                exynos_dsi_set_cmd_lpm(dsi, xfer->flags & MIPI_DSI_MSG_USE_LPM);
+                dsi->state ^= DSIM_STATE_CMD_LPM;
+        }
+
+        writel(reg, dsi->reg_base + DSIM_PKTHDR_REG);
+
+        if (xfer->flags & MIPI_DSI_MSG_REQ_ACK)
+                exynos_dsi_force_bta(dsi);
+}
+
+static void exynos_dsi_read_from_fifo(struct exynos_dsi *dsi,
+                                        struct exynos_dsi_transfer *xfer)
+{
+        u8 *payload = xfer->rx_payload + xfer->rx_done;
+        bool first = !xfer->rx_done;
+        struct device *dev = dsi->dev;
+        u16 length;
+        u32 reg;
+
+        if (first) {
+                reg = readl(dsi->reg_base + DSIM_RXFIFO_REG);
+
+                switch (reg & 0x3f) {
+                case MIPI_DSI_RX_GENERIC_SHORT_READ_RESPONSE_2BYTE:
+                case MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_2BYTE:
+                        if (xfer->rx_len >= 2) {
+                                payload[1] = reg >> 16;
+                                ++xfer->rx_done;
+                        }
+                        /* Fall through */
+                case MIPI_DSI_RX_GENERIC_SHORT_READ_RESPONSE_1BYTE:
+                case MIPI_DSI_RX_DCS_SHORT_READ_RESPONSE_1BYTE:
+                        payload[0] = reg >> 8;
+                        ++xfer->rx_done;
+                        xfer->rx_len = xfer->rx_done;
+                        xfer->result = 0;
+                        goto clear_fifo;
+                case MIPI_DSI_RX_ACKNOWLEDGE_AND_ERROR_REPORT:
+                        dev_err(dev, "DSI Error Report: 0x%04x\n",
+                                (reg >> 8) & 0xffff);
+                        xfer->result = 0;
+                        goto clear_fifo;
+                }
+
+                length = (reg >> 8) & 0xffff;
+                if (length > xfer->rx_len) {
+                        dev_err(dev,
+                                "response too long (%u > %u bytes), stripping\n",
+                                xfer->rx_len, length);
+                        length = xfer->rx_len;
+                } else if (length < xfer->rx_len)
+                        xfer->rx_len = length;
+        }
+
+        length = xfer->rx_len - xfer->rx_done;
+        xfer->rx_done += length;
+
+        /* Receive payload */
+        while (length >= 4) {
+                reg = readl(dsi->reg_base + DSIM_RXFIFO_REG);
+                payload[0] = (reg >>  0) & 0xff;
+                payload[1] = (reg >>  8) & 0xff;
+                payload[2] = (reg >> 16) & 0xff;
+                payload[3] = (reg >> 24) & 0xff;
+                payload += 4;
+                length -= 4;
+        }
+
+        if (length) {
+                reg = readl(dsi->reg_base + DSIM_RXFIFO_REG);
+                switch (length) {
+                case 3:
+                        payload[2] = (reg >> 16) & 0xff;
+                        /* Fall through */
+                case 2:
+                        payload[1] = (reg >> 8) & 0xff;
+                        /* Fall through */
+                case 1:
+                        payload[0] = reg & 0xff;
+                }
+        }
+
+        if (xfer->rx_done == xfer->rx_len)
+                xfer->result = 0;
+
+clear_fifo:
+        length = DSI_RX_FIFO_SIZE / 4;
+        do {
+                reg = readl(dsi->reg_base + DSIM_RXFIFO_REG);
+                if (reg == DSI_RX_FIFO_EMPTY)
+                        break;
+        } while (--length);
+}
+
+static void exynos_dsi_transfer_start(struct exynos_dsi *dsi)
+{
+        unsigned long flags;
+        struct exynos_dsi_transfer *xfer;
+        bool start = false;
+
+again:
+        spin_lock_irqsave(&dsi->transfer_lock, flags);
+
+        if (list_empty(&dsi->transfer_list)) {
+                spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+                return;
+        }
+
+        xfer = list_first_entry(&dsi->transfer_list,
+                                        struct exynos_dsi_transfer, list);
+
+        spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+
+        if (xfer->tx_len && xfer->tx_done == xfer->tx_len)
+                /* waiting for RX */
+                return;
+
+        exynos_dsi_send_to_fifo(dsi, xfer);
+
+        if (xfer->tx_len || xfer->rx_len)
+                return;
+
+        xfer->result = 0;
+        complete(&xfer->completed);
+
+        spin_lock_irqsave(&dsi->transfer_lock, flags);
+
+        list_del_init(&xfer->list);
+        start = !list_empty(&dsi->transfer_list);
+
+        spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+
+        if (start)
+                goto again;
+}
+
+static bool exynos_dsi_transfer_finish(struct exynos_dsi *dsi)
+{
+        struct exynos_dsi_transfer *xfer;
+        unsigned long flags;
+        bool start = true;
+
+        spin_lock_irqsave(&dsi->transfer_lock, flags);
+
+        if (list_empty(&dsi->transfer_list)) {
+                spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+                return false;
+        }
+
+        xfer = list_first_entry(&dsi->transfer_list,
+                                        struct exynos_dsi_transfer, list);
+
+        spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+
+        dev_dbg(dsi->dev,
+                "> xfer %p, tx_len %u, tx_done %u, rx_len %u, rx_done %u\n",
+                xfer, xfer->tx_len, xfer->tx_done, xfer->rx_len, xfer->rx_done);
+
+        if (xfer->tx_done != xfer->tx_len)
+                return true;
+
+        if (xfer->rx_done != xfer->rx_len)
+                exynos_dsi_read_from_fifo(dsi, xfer);
+
+        if (xfer->rx_done != xfer->rx_len)
+                return true;
+
+        spin_lock_irqsave(&dsi->transfer_lock, flags);
+
+        list_del_init(&xfer->list);
+        start = !list_empty(&dsi->transfer_list);
+
+        spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+
+        if (!xfer->rx_len)
+                xfer->result = 0;
+        complete(&xfer->completed);
+
+        return start;
+}
+
+static void exynos_dsi_remove_transfer(struct exynos_dsi *dsi,
+                                        struct exynos_dsi_transfer *xfer)
+{
+        unsigned long flags;
+        bool start;
+
+        spin_lock_irqsave(&dsi->transfer_lock, flags);
+
+        if (!list_empty(&dsi->transfer_list) &&
+            xfer == list_first_entry(&dsi->transfer_list,
+                                     struct exynos_dsi_transfer, list)) {
+                list_del_init(&xfer->list);
+                start = !list_empty(&dsi->transfer_list);
+                spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+                if (start)
+                        exynos_dsi_transfer_start(dsi);
+                return;
+        }
+
+        list_del_init(&xfer->list);
+
+        spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+}
+
+static int exynos_dsi_transfer(struct exynos_dsi *dsi,
+                                        struct exynos_dsi_transfer *xfer)
+{
+        unsigned long flags;
+        bool stopped;
+
+        xfer->tx_done = 0;
+        xfer->rx_done = 0;
+        xfer->result = -ETIMEDOUT;
+        init_completion(&xfer->completed);
+
+        spin_lock_irqsave(&dsi->transfer_lock, flags);
+
+        stopped = list_empty(&dsi->transfer_list);
+        list_add_tail(&xfer->list, &dsi->transfer_list);
+
+        spin_unlock_irqrestore(&dsi->transfer_lock, flags);
+
+        if (stopped)
+                exynos_dsi_transfer_start(dsi);
+
+        wait_for_completion_timeout(&xfer->completed,
+                                    msecs_to_jiffies(DSI_XFER_TIMEOUT_MS));
+        if (xfer->result == -ETIMEDOUT) {
+                exynos_dsi_remove_transfer(dsi, xfer);
+                dev_err(dsi->dev, "xfer timed out: %*ph %*ph\n", 2, xfer->data,
+                        xfer->tx_len, xfer->tx_payload);
+                return -ETIMEDOUT;
+        }
+
+        /* Also covers hardware timeout condition */
+        return xfer->result;
+}
+
+static irqreturn_t exynos_dsi_irq(int irq, void *dev_id)
+{
+        struct exynos_dsi *dsi = dev_id;
+        u32 status;
+
+        status = readl(dsi->reg_base + DSIM_INTSRC_REG);
+        if (!status) {
+                static unsigned long int j;
+                if (printk_timed_ratelimit(&j, 500))
+                        dev_warn(dsi->dev, "spurious interrupt\n");
+                return IRQ_HANDLED;
+        }
+        writel(status, dsi->reg_base + DSIM_INTSRC_REG);
+
+        if (status & DSIM_INT_SW_RST_RELEASE) {
+                u32 mask = ~(DSIM_INT_RX_DONE | DSIM_INT_SFR_FIFO_EMPTY);
+                writel(mask, dsi->reg_base + DSIM_INTMSK_REG);
+                complete(&dsi->completed);
+                return IRQ_HANDLED;
+        }
+
+        if (!(status & (DSIM_INT_RX_DONE | DSIM_INT_SFR_FIFO_EMPTY)))
+                return IRQ_HANDLED;
+
+        if (exynos_dsi_transfer_finish(dsi))
+                exynos_dsi_transfer_start(dsi);
+
+        return IRQ_HANDLED;
+}
+
+static int exynos_dsi_init(struct exynos_dsi *dsi)
+{
+        exynos_dsi_enable_clock(dsi);
+        exynos_dsi_reset(dsi);
+        enable_irq(dsi->irq);
+        exynos_dsi_wait_for_reset(dsi);
+        exynos_dsi_init_link(dsi);
+
+        return 0;
+}
+
+static int exynos_dsi_host_attach(struct mipi_dsi_host *host,
+                                  struct mipi_dsi_device *device)
+{
+        struct exynos_dsi *dsi = host_to_dsi(host);
+
+        dsi->lanes = device->lanes;
+        dsi->format = device->format;
+        dsi->mode_flags = device->mode_flags;
+        dsi->panel_node = device->dev.of_node;
+
+        if (dsi->connector.dev)
+                drm_helper_hpd_irq_event(dsi->connector.dev);
+
+        return 0;
+}
+
+static int exynos_dsi_host_detach(struct mipi_dsi_host *host,
+                                  struct mipi_dsi_device *device)
+{
+        struct exynos_dsi *dsi = host_to_dsi(host);
+
+        dsi->panel_node = NULL;
+
+        if (dsi->connector.dev)
+                drm_helper_hpd_irq_event(dsi->connector.dev);
+
+        return 0;
+}
+
+/* distinguish between short and long DSI packet types */
+static bool exynos_dsi_is_short_dsi_type(u8 type)
+{
+        return (type & 0x0f) <= 8;
+}
+
+static ssize_t exynos_dsi_host_transfer(struct mipi_dsi_host *host,
+                                       struct mipi_dsi_msg *msg)
+{
+        struct exynos_dsi *dsi = host_to_dsi(host);
+        struct exynos_dsi_transfer xfer;
+        int ret;
+
+        if (!(dsi->state & DSIM_STATE_INITIALIZED)) {
+                ret = exynos_dsi_init(dsi);
+                if (ret)
+                        return ret;
+                dsi->state |= DSIM_STATE_INITIALIZED;
+        }
+
+        if (msg->tx_len == 0)
+                return -EINVAL;
+
+        xfer.data_id = msg->type | (msg->channel << 6);
+
+        if (exynos_dsi_is_short_dsi_type(msg->type)) {
+                const char *tx_buf = msg->tx_buf;
+
+                if (msg->tx_len > 2)
+                        return -EINVAL;
+                xfer.tx_len = 0;
+                xfer.data[0] = tx_buf[0];
+                xfer.data[1] = (msg->tx_len == 2) ? tx_buf[1] : 0;
+        } else {
+                xfer.tx_len = msg->tx_len;
+                xfer.data[0] = msg->tx_len & 0xff;
+                xfer.data[1] = msg->tx_len >> 8;
+                xfer.tx_payload = msg->tx_buf;
+        }
+
+        xfer.rx_len = msg->rx_len;
+        xfer.rx_payload = msg->rx_buf;
+        xfer.flags = msg->flags;
+
+        ret = exynos_dsi_transfer(dsi, &xfer);
+        return (ret < 0) ? ret : xfer.rx_done;
+}
+
+static const struct mipi_dsi_host_ops exynos_dsi_ops = {
+        .attach = exynos_dsi_host_attach,
+        .detach = exynos_dsi_host_detach,
+        .transfer = exynos_dsi_host_transfer,
+};
+
+static int exynos_dsi_poweron(struct exynos_dsi *dsi)
+{
+        int ret;
+
+        ret = regulator_bulk_enable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
+        if (ret < 0) {
+                dev_err(dsi->dev, "cannot enable regulators %d\n", ret);
+                return ret;
+        }
+
+        ret = clk_prepare_enable(dsi->bus_clk);
+        if (ret < 0) {
+                dev_err(dsi->dev, "cannot enable bus clock %d\n", ret);
+                goto err_bus_clk;
+        }
+
+        ret = clk_prepare_enable(dsi->pll_clk);
+        if (ret < 0) {
+                dev_err(dsi->dev, "cannot enable pll clock %d\n", ret);
+                goto err_pll_clk;
+        }
+
+        ret = phy_power_on(dsi->phy);
+        if (ret < 0) {
+                dev_err(dsi->dev, "cannot enable phy %d\n", ret);
+                goto err_phy;
+        }
+
+        return 0;
+
+err_phy:
+        clk_disable_unprepare(dsi->pll_clk);
+err_pll_clk:
+        clk_disable_unprepare(dsi->bus_clk);
+err_bus_clk:
+        regulator_bulk_disable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
+
+        return ret;
+}
+
+static void exynos_dsi_poweroff(struct exynos_dsi *dsi)
+{
+        int ret;
+
+        usleep_range(10000, 20000);
+
+        if (dsi->state & DSIM_STATE_INITIALIZED) {
+                dsi->state &= ~DSIM_STATE_INITIALIZED;
+
+                exynos_dsi_disable_clock(dsi);
+
+                disable_irq(dsi->irq);
+        }
+
+        dsi->state &= ~DSIM_STATE_CMD_LPM;
+
+        phy_power_off(dsi->phy);
+
+        clk_disable_unprepare(dsi->pll_clk);
+        clk_disable_unprepare(dsi->bus_clk);
+
+        ret = regulator_bulk_disable(ARRAY_SIZE(dsi->supplies), dsi->supplies);
+        if (ret < 0)
+                dev_err(dsi->dev, "cannot disable regulators %d\n", ret);
+}
+
+static int exynos_dsi_enable(struct exynos_dsi *dsi)
+{
+        int ret;
+
+        if (dsi->state & DSIM_STATE_ENABLED)
+                return 0;
+
+        ret = exynos_dsi_poweron(dsi);
+        if (ret < 0)
+                return ret;
+
+        ret = drm_panel_enable(dsi->panel);
+        if (ret < 0) {
+                exynos_dsi_poweroff(dsi);
+                return ret;
+        }
+
+        exynos_dsi_set_display_mode(dsi);
+        exynos_dsi_set_display_enable(dsi, true);
+
+        dsi->state |= DSIM_STATE_ENABLED;
+
+        return 0;
+}
+
+static void exynos_dsi_disable(struct exynos_dsi *dsi)
+{
+        if (!(dsi->state & DSIM_STATE_ENABLED))
+                return;
+
+        exynos_dsi_set_display_enable(dsi, false);
+        drm_panel_disable(dsi->panel);
+        exynos_dsi_poweroff(dsi);
+
+        dsi->state &= ~DSIM_STATE_ENABLED;
+}
+
+static void exynos_dsi_dpms(struct exynos_drm_display *display, int mode)
+{
+        struct exynos_dsi *dsi = display->ctx;
+
+        if (dsi->panel) {
+                switch (mode) {
+                case DRM_MODE_DPMS_ON:
+                        exynos_dsi_enable(dsi);
+                        break;
+                case DRM_MODE_DPMS_STANDBY:
+                case DRM_MODE_DPMS_SUSPEND:
+                case DRM_MODE_DPMS_OFF:
+                        exynos_dsi_disable(dsi);
+                        break;
+                default:
+                        break;
+                }
+        }
+}
+
+static enum drm_connector_status
+exynos_dsi_detect(struct drm_connector *connector, bool force)
+{
+        struct exynos_dsi *dsi = connector_to_dsi(connector);
+
+        if (!dsi->panel) {
+                dsi->panel = of_drm_find_panel(dsi->panel_node);
+                if (dsi->panel)
+                        drm_panel_attach(dsi->panel, &dsi->connector);
+        } else if (!dsi->panel_node) {
+                struct exynos_drm_display *display;
+
+                display = platform_get_drvdata(to_platform_device(dsi->dev));
+                exynos_dsi_dpms(display, DRM_MODE_DPMS_OFF);
+                drm_panel_detach(dsi->panel);
+                dsi->panel = NULL;
+        }
+
+        if (dsi->panel)
+                return connector_status_connected;
+
+        return connector_status_disconnected;
+}
+
+static void exynos_dsi_connector_destroy(struct drm_connector *connector)
+{
+}
+
+static struct drm_connector_funcs exynos_dsi_connector_funcs = {
+        .dpms = drm_helper_connector_dpms,
+        .detect = exynos_dsi_detect,
+        .fill_modes = drm_helper_probe_single_connector_modes,
+        .destroy = exynos_dsi_connector_destroy,
+};
+
+static int exynos_dsi_get_modes(struct drm_connector *connector)
+{
+        struct exynos_dsi *dsi = connector_to_dsi(connector);
+
+        if (dsi->panel)
+                return dsi->panel->funcs->get_modes(dsi->panel);
+
+        return 0;
+}
+
+static int exynos_dsi_mode_valid(struct drm_connector *connector,
+                                 struct drm_display_mode *mode)
+{
+        return MODE_OK;
+}
+
+static struct drm_encoder *
+exynos_dsi_best_encoder(struct drm_connector *connector)
+{
+        struct exynos_dsi *dsi = connector_to_dsi(connector);
+
+        return dsi->encoder;
+}
+
+static struct drm_connector_helper_funcs exynos_dsi_connector_helper_funcs = {
+        .get_modes = exynos_dsi_get_modes,
+        .mode_valid = exynos_dsi_mode_valid,
+        .best_encoder = exynos_dsi_best_encoder,
+};
+
+static int exynos_dsi_create_connector(struct exynos_drm_display *display,
+                                       struct drm_encoder *encoder)
+{
+        struct exynos_dsi *dsi = display->ctx;
+        struct drm_connector *connector = &dsi->connector;
+        int ret;
+
+        dsi->encoder = encoder;
+
+        connector->polled = DRM_CONNECTOR_POLL_HPD;
+
+        ret = drm_connector_init(encoder->dev, connector,
+                                 &exynos_dsi_connector_funcs,
+                                 DRM_MODE_CONNECTOR_DSI);
+        if (ret) {
+                DRM_ERROR("Failed to initialize connector with drm\n");
+                return ret;
+        }
+
+        drm_connector_helper_add(connector, &exynos_dsi_connector_helper_funcs);
+        drm_sysfs_connector_add(connector);
+        drm_mode_connector_attach_encoder(connector, encoder);
+
+        return 0;
+}
+
+static void exynos_dsi_mode_set(struct exynos_drm_display *display,
+                         struct drm_display_mode *mode)
+{
+        struct exynos_dsi *dsi = display->ctx;
+        struct videomode *vm = &dsi->vm;
+
+        vm->hactive = mode->hdisplay;
+        vm->vactive = mode->vdisplay;
+        vm->vfront_porch = mode->vsync_start - mode->vdisplay;
+        vm->vback_porch = mode->vtotal - mode->vsync_end;
+        vm->vsync_len = mode->vsync_end - mode->vsync_start;
+        vm->hfront_porch = mode->hsync_start - mode->hdisplay;
+        vm->hback_porch = mode->htotal - mode->hsync_end;
+        vm->hsync_len = mode->hsync_end - mode->hsync_start;
+}
+
+static struct exynos_drm_display_ops exynos_dsi_display_ops = {
+        .create_connector = exynos_dsi_create_connector,
+        .mode_set = exynos_dsi_mode_set,
+        .dpms = exynos_dsi_dpms
+};
+
+static struct exynos_drm_display exynos_dsi_display = {
+        .type = EXYNOS_DISPLAY_TYPE_LCD,
+        .ops = &exynos_dsi_display_ops,
+};
+
+/* of_* functions will be removed after merge of of_graph patches */
+static struct device_node *
+of_get_child_by_name_reg(struct device_node *parent, const char *name, u32 reg)
+{
+        struct device_node *np;
+
+        for_each_child_of_node(parent, np) {
+                u32 r;
+
+                if (!np->name || of_node_cmp(np->name, name))
+                        continue;
+
+                if (of_property_read_u32(np, "reg", &r) < 0)
+                        r = 0;
+
+                if (reg == r)
+                        break;
+        }
+
+        return np;
+}
+
+static struct device_node *of_graph_get_port_by_reg(struct device_node *parent,
+                                                    u32 reg)
+{
+        struct device_node *ports, *port;
+
+        ports = of_get_child_by_name(parent, "ports");
+        if (ports)
+                parent = ports;
+
+        port = of_get_child_by_name_reg(parent, "port", reg);
+
+        of_node_put(ports);
+
+        return port;
+}
+
+static struct device_node *
+of_graph_get_endpoint_by_reg(struct device_node *port, u32 reg)
+{
+        return of_get_child_by_name_reg(port, "endpoint", reg);
+}
+
+static int exynos_dsi_of_read_u32(const struct device_node *np,
+                                  const char *propname, u32 *out_value)
+{
+        int ret = of_property_read_u32(np, propname, out_value);
+
+        if (ret < 0)
+                pr_err("%s: failed to get '%s' property\n", np->full_name,
+                       propname);
+
+        return ret;
+}
+
+enum {
+        DSI_PORT_IN,
+        DSI_PORT_OUT
+};
+
+static int exynos_dsi_parse_dt(struct exynos_dsi *dsi)
+{
+        struct device *dev = dsi->dev;
+        struct device_node *node = dev->of_node;
+        struct device_node *port, *ep;
+        int ret;
+
+        ret = exynos_dsi_of_read_u32(node, "samsung,pll-clock-frequency",
+                                     &dsi->pll_clk_rate);
+        if (ret < 0)
+                return ret;
+
+        port = of_graph_get_port_by_reg(node, DSI_PORT_OUT);
+        if (!port) {
+                dev_err(dev, "no output port specified\n");
+                return -EINVAL;
+        }
+
+        ep = of_graph_get_endpoint_by_reg(port, 0);
+        of_node_put(port);
+        if (!ep) {
+                dev_err(dev, "no endpoint specified in output port\n");
+                return -EINVAL;
+        }
+
+        ret = exynos_dsi_of_read_u32(ep, "samsung,burst-clock-frequency",
+                                     &dsi->burst_clk_rate);
+        if (ret < 0)
+                goto end;
+
+        ret = exynos_dsi_of_read_u32(ep, "samsung,esc-clock-frequency",
+                                     &dsi->esc_clk_rate);
+
+end:
+        of_node_put(ep);
+
+        return ret;
+}
+
+static int exynos_dsi_probe(struct platform_device *pdev)
+{
+        struct resource *res;
+        struct exynos_dsi *dsi;
+        int ret;
+
+        dsi = devm_kzalloc(&pdev->dev, sizeof(*dsi), GFP_KERNEL);
+        if (!dsi) {
+                dev_err(&pdev->dev, "failed to allocate dsi object.\n");
+                return -ENOMEM;
+        }
+
+        init_completion(&dsi->completed);
+        spin_lock_init(&dsi->transfer_lock);
+        INIT_LIST_HEAD(&dsi->transfer_list);
+
+        dsi->dsi_host.ops = &exynos_dsi_ops;
+        dsi->dsi_host.dev = &pdev->dev;
+
+        dsi->dev = &pdev->dev;
+
+        ret = exynos_dsi_parse_dt(dsi);
+        if (ret)
+                return ret;
+
+        dsi->supplies[0].supply = "vddcore";
+        dsi->supplies[1].supply = "vddio";
+        ret = devm_regulator_bulk_get(&pdev->dev, ARRAY_SIZE(dsi->supplies),
+                                      dsi->supplies);
+        if (ret) {
+                dev_info(&pdev->dev, "failed to get regulators: %d\n", ret);
+                return -EPROBE_DEFER;
+        }
+
+        dsi->pll_clk = devm_clk_get(&pdev->dev, "pll_clk");
+        if (IS_ERR(dsi->pll_clk)) {
+                dev_info(&pdev->dev, "failed to get dsi pll input clock\n");
+                return -EPROBE_DEFER;
+        }
+
+        dsi->bus_clk = devm_clk_get(&pdev->dev, "bus_clk");
+        if (IS_ERR(dsi->bus_clk)) {
+                dev_info(&pdev->dev, "failed to get dsi bus clock\n");
+                return -EPROBE_DEFER;
+        }
+
+        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        dsi->reg_base = devm_ioremap_resource(&pdev->dev, res);
+        if (!dsi->reg_base) {
+                dev_err(&pdev->dev, "failed to remap io region\n");
+                return -EADDRNOTAVAIL;
+        }
+
+        dsi->phy = devm_phy_get(&pdev->dev, "dsim");
+        if (IS_ERR(dsi->phy)) {
+                dev_info(&pdev->dev, "failed to get dsim phy\n");
+                return -EPROBE_DEFER;
+        }
+
+        dsi->irq = platform_get_irq(pdev, 0);
+        if (dsi->irq < 0) {
+                dev_err(&pdev->dev, "failed to request dsi irq resource\n");
+                return dsi->irq;
+        }
+
+        irq_set_status_flags(dsi->irq, IRQ_NOAUTOEN);
+        ret = devm_request_threaded_irq(&pdev->dev, dsi->irq, NULL,
+                                        exynos_dsi_irq, IRQF_ONESHOT,
+                                        dev_name(&pdev->dev), dsi);
+        if (ret) {
+                dev_err(&pdev->dev, "failed to request dsi irq\n");
+                return ret;
+        }
+
+        exynos_dsi_display.ctx = dsi;
+
+        platform_set_drvdata(pdev, &exynos_dsi_display);
+        exynos_drm_display_register(&exynos_dsi_display);
+
+        return mipi_dsi_host_register(&dsi->dsi_host);
+}
+
+static int exynos_dsi_remove(struct platform_device *pdev)
+{
+        struct exynos_dsi *dsi = exynos_dsi_display.ctx;
+
+        exynos_dsi_dpms(&exynos_dsi_display, DRM_MODE_DPMS_OFF);
+
+        exynos_drm_display_unregister(&exynos_dsi_display);
+        mipi_dsi_host_unregister(&dsi->dsi_host);
+
+        return 0;
+}
+
+#if CONFIG_PM_SLEEP
+static int exynos_dsi_resume(struct device *dev)
+{
+        struct exynos_dsi *dsi = exynos_dsi_display.ctx;
+
+        if (dsi->state & DSIM_STATE_ENABLED) {
+                dsi->state &= ~DSIM_STATE_ENABLED;
+                exynos_dsi_enable(dsi);
+        }
+
+        return 0;
+}
+
+static int exynos_dsi_suspend(struct device *dev)
+{
+        struct exynos_dsi *dsi = exynos_dsi_display.ctx;
+
+        if (dsi->state & DSIM_STATE_ENABLED) {
+                exynos_dsi_disable(dsi);
+                dsi->state |= DSIM_STATE_ENABLED;
+        }
+
+        return 0;
+}
+#endif
+
+static const struct dev_pm_ops exynos_dsi_pm_ops = {
+        SET_SYSTEM_SLEEP_PM_OPS(exynos_dsi_suspend, exynos_dsi_resume)
+};
+
+static struct of_device_id exynos_dsi_of_match[] = {
+        { .compatible = "samsung,exynos4210-mipi-dsi" },
+        { }
+};
+
+struct platform_driver dsi_driver = {
+        .probe = exynos_dsi_probe,
+        .remove = exynos_dsi_remove,
+        .driver = {
+                   .name = "exynos-dsi",
+                   .owner = THIS_MODULE,
+                   .pm = &exynos_dsi_pm_ops,
+                   .of_match_table = exynos_dsi_of_match,
+        },
+};
+
+MODULE_AUTHOR("Tomasz Figa <t.figa@samsung.com>");
+MODULE_AUTHOR("Andrzej Hajda <a.hajda@samsung.com>");
+MODULE_DESCRIPTION("Samsung SoC MIPI DSI Master");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/exynos/exynos_drm_encoder.c b/drivers/gpu/drm/exynos/exynos_drm_encoder.c
index 06f1b2a09da7..7e282e3d6038 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_encoder.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_encoder.c
@@ -17,7 +17,6 @@
 
 #include "exynos_drm_drv.h"
 #include "exynos_drm_encoder.h"
-#include "exynos_drm_connector.h"
 
 #define to_exynos_encoder(x)    container_of(x, struct exynos_drm_encoder,\
                                 drm_encoder)
@@ -26,72 +25,22 @@
  * exynos specific encoder structure.
  *
  * @drm_encoder: encoder object.
- * @manager: specific encoder has its own manager to control a hardware
- *      appropriately and we can access a hardware drawing on this manager.
- * @dpms: store the encoder dpms value.
- * @updated: indicate whether overlay data updating is needed or not.
+ * @display: the display structure that maps to this encoder
  */
 struct exynos_drm_encoder {
-        struct drm_crtc                 *old_crtc;
         struct drm_encoder              drm_encoder;
-        struct exynos_drm_manager       *manager;
-        int                             dpms;
-        bool                            updated;
+        struct exynos_drm_display       *display;
 };
 
-static void exynos_drm_connector_power(struct drm_encoder *encoder, int mode)
-{
-        struct drm_device *dev = encoder->dev;
-        struct drm_connector *connector;
-
-        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                if (exynos_drm_best_encoder(connector) == encoder) {
-                        DRM_DEBUG_KMS("connector[%d] dpms[%d]\n",
-                                        connector->base.id, mode);
-
-                        exynos_drm_display_power(connector, mode);
-                }
-        }
-}
-
 static void exynos_drm_encoder_dpms(struct drm_encoder *encoder, int mode)
 {
-        struct drm_device *dev = encoder->dev;
-        struct exynos_drm_manager *manager = exynos_drm_get_manager(encoder);
-        struct exynos_drm_manager_ops *manager_ops = manager->ops;
         struct exynos_drm_encoder *exynos_encoder = to_exynos_encoder(encoder);
+        struct exynos_drm_display *display = exynos_encoder->display;
 
         DRM_DEBUG_KMS("encoder dpms: %d\n", mode);
 
-        if (exynos_encoder->dpms == mode) {
-                DRM_DEBUG_KMS("desired dpms mode is same as previous one.\n");
-                return;
-        }
-
-        mutex_lock(&dev->struct_mutex);
-
-        switch (mode) {
-        case DRM_MODE_DPMS_ON:
-                if (manager_ops && manager_ops->apply)
-                        if (!exynos_encoder->updated)
-                                manager_ops->apply(manager->dev);
-
-                exynos_drm_connector_power(encoder, mode);
-                exynos_encoder->dpms = mode;
-                break;
-        case DRM_MODE_DPMS_STANDBY:
-        case DRM_MODE_DPMS_SUSPEND:
-        case DRM_MODE_DPMS_OFF:
-                exynos_drm_connector_power(encoder, mode);
-                exynos_encoder->dpms = mode;
-                exynos_encoder->updated = false;
-                break;
-        default:
-                DRM_ERROR("unspecified mode %d\n", mode);
-                break;
-        }
-
-        mutex_unlock(&dev->struct_mutex);
+        if (display->ops->dpms)
+                display->ops->dpms(display, mode);
 }
 
 static bool
@@ -100,87 +49,31 @@ exynos_drm_encoder_mode_fixup(struct drm_encoder *encoder,
                                struct drm_display_mode *adjusted_mode)
 {
         struct drm_device *dev = encoder->dev;
+        struct exynos_drm_encoder *exynos_encoder = to_exynos_encoder(encoder);
+        struct exynos_drm_display *display = exynos_encoder->display;
         struct drm_connector *connector;
-        struct exynos_drm_manager *manager = exynos_drm_get_manager(encoder);
-        struct exynos_drm_manager_ops *manager_ops = manager->ops;
 
         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                if (connector->encoder == encoder)
-                        if (manager_ops && manager_ops->mode_fixup)
-                                manager_ops->mode_fixup(manager->dev, connector,
-                                                        mode, adjusted_mode);
+                if (connector->encoder != encoder)
+                        continue;
+
+                if (display->ops->mode_fixup)
+                        display->ops->mode_fixup(display, connector, mode,
+                                        adjusted_mode);
         }
 
         return true;
 }
 
-static void disable_plane_to_crtc(struct drm_device *dev,
-                                                struct drm_crtc *old_crtc,
-                                                struct drm_crtc *new_crtc)
-{
-        struct drm_plane *plane;
-
-        /*
-         * if old_crtc isn't same as encoder->crtc then it means that
-         * user changed crtc id to another one so the plane to old_crtc
-         * should be disabled and plane->crtc should be set to new_crtc
-         * (encoder->crtc)
-         */
-        list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
-                if (plane->crtc == old_crtc) {
-                        /*
-                         * do not change below call order.
-                         *
-                         * plane->funcs->disable_plane call checks
-                         * if encoder->crtc is same as plane->crtc and if same
-                         * then overlay_ops->disable callback will be called
-                         * to diasble current hw overlay so plane->crtc should
-                         * have new_crtc because new_crtc was set to
-                         * encoder->crtc in advance.
-                         */
-                        plane->crtc = new_crtc;
-                        plane->funcs->disable_plane(plane);
-                }
-        }
-}
-
 static void exynos_drm_encoder_mode_set(struct drm_encoder *encoder,
                                          struct drm_display_mode *mode,
                                          struct drm_display_mode *adjusted_mode)
 {
-        struct drm_device *dev = encoder->dev;
-        struct drm_connector *connector;
-        struct exynos_drm_manager *manager;
-        struct exynos_drm_manager_ops *manager_ops;
-
-        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                if (connector->encoder == encoder) {
-                        struct exynos_drm_encoder *exynos_encoder;
-
-                        exynos_encoder = to_exynos_encoder(encoder);
-
-                        if (exynos_encoder->old_crtc != encoder->crtc &&
-                                        exynos_encoder->old_crtc) {
-
-                                /*
-                                 * disable a plane to old crtc and change
-                                 * crtc of the plane to new one.
-                                 */
-                                disable_plane_to_crtc(dev,
-                                                exynos_encoder->old_crtc,
-                                                encoder->crtc);
-                        }
-
-                        manager = exynos_drm_get_manager(encoder);
-                        manager_ops = manager->ops;
-
-                        if (manager_ops && manager_ops->mode_set)
-                                manager_ops->mode_set(manager->dev,
-                                                        adjusted_mode);
+        struct exynos_drm_encoder *exynos_encoder = to_exynos_encoder(encoder);
+        struct exynos_drm_display *display = exynos_encoder->display;
 
-                        exynos_encoder->old_crtc = encoder->crtc;
-                }
-        }
+        if (display->ops->mode_set)
+                display->ops->mode_set(display, adjusted_mode);
 }
 
 static void exynos_drm_encoder_prepare(struct drm_encoder *encoder)
@@ -191,53 +84,15 @@ static void exynos_drm_encoder_prepare(struct drm_encoder *encoder)
 static void exynos_drm_encoder_commit(struct drm_encoder *encoder)
 {
         struct exynos_drm_encoder *exynos_encoder = to_exynos_encoder(encoder);
-        struct exynos_drm_manager *manager = exynos_encoder->manager;
-        struct exynos_drm_manager_ops *manager_ops = manager->ops;
-
-        if (manager_ops && manager_ops->commit)
-                manager_ops->commit(manager->dev);
-
-        /*
-         * this will avoid one issue that overlay data is updated to
-         * real hardware two times.
-         * And this variable will be used to check if the data was
-         * already updated or not by exynos_drm_encoder_dpms function.
-         */
-        exynos_encoder->updated = true;
-
-        /*
-         * In case of setcrtc, there is no way to update encoder's dpms
-         * so update it here.
-         */
-        exynos_encoder->dpms = DRM_MODE_DPMS_ON;
-}
+        struct exynos_drm_display *display = exynos_encoder->display;
 
-void exynos_drm_encoder_complete_scanout(struct drm_framebuffer *fb)
-{
-        struct exynos_drm_encoder *exynos_encoder;
-        struct exynos_drm_manager_ops *ops;
-        struct drm_device *dev = fb->dev;
-        struct drm_encoder *encoder;
+        if (display->ops->dpms)
+                display->ops->dpms(display, DRM_MODE_DPMS_ON);
 
-        /*
-         * make sure that overlay data are updated to real hardware
-         * for all encoders.
-         */
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-                exynos_encoder = to_exynos_encoder(encoder);
-                ops = exynos_encoder->manager->ops;
-
-                /*
-                 * wait for vblank interrupt
-                 * - this makes sure that overlay data are updated to
-                 *      real hardware.
-                 */
-                if (ops->wait_for_vblank)
-                        ops->wait_for_vblank(exynos_encoder->manager->dev);
-        }
+        if (display->ops->commit)
+                display->ops->commit(display);
 }
 
-
 static void exynos_drm_encoder_disable(struct drm_encoder *encoder)
 {
         struct drm_plane *plane;
@@ -246,7 +101,7 @@ static void exynos_drm_encoder_disable(struct drm_encoder *encoder)
         exynos_drm_encoder_dpms(encoder, DRM_MODE_DPMS_OFF);
 
         /* all planes connected to this encoder should be also disabled. */
-        list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
+        drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) {
                 if (plane->crtc == encoder->crtc)
                         plane->funcs->disable_plane(plane);
         }
@@ -263,10 +118,7 @@ static struct drm_encoder_helper_funcs exynos_encoder_helper_funcs = {
 
 static void exynos_drm_encoder_destroy(struct drm_encoder *encoder)
 {
-        struct exynos_drm_encoder *exynos_encoder =
-                to_exynos_encoder(encoder);
-
-        exynos_encoder->manager->pipe = -1;
+        struct exynos_drm_encoder *exynos_encoder = to_exynos_encoder(encoder);
 
         drm_encoder_cleanup(encoder);
         kfree(exynos_encoder);
@@ -281,13 +133,12 @@ static unsigned int exynos_drm_encoder_clones(struct drm_encoder *encoder)
         struct drm_encoder *clone;
         struct drm_device *dev = encoder->dev;
         struct exynos_drm_encoder *exynos_encoder = to_exynos_encoder(encoder);
-        struct exynos_drm_display_ops *display_ops =
-                                exynos_encoder->manager->display_ops;
+        struct exynos_drm_display *display = exynos_encoder->display;
         unsigned int clone_mask = 0;
         int cnt = 0;
 
         list_for_each_entry(clone, &dev->mode_config.encoder_list, head) {
-                switch (display_ops->type) {
+                switch (display->type) {
                 case EXYNOS_DISPLAY_TYPE_LCD:
                 case EXYNOS_DISPLAY_TYPE_HDMI:
                 case EXYNOS_DISPLAY_TYPE_VIDI:
@@ -311,24 +162,20 @@ void exynos_drm_encoder_setup(struct drm_device *dev)
 
 struct drm_encoder *
 exynos_drm_encoder_create(struct drm_device *dev,
-                           struct exynos_drm_manager *manager,
-                           unsigned int possible_crtcs)
+                           struct exynos_drm_display *display,
+                           unsigned long possible_crtcs)
 {
         struct drm_encoder *encoder;
         struct exynos_drm_encoder *exynos_encoder;
 
-        if (!manager || !possible_crtcs)
-                return NULL;
-
-        if (!manager->dev)
+        if (!possible_crtcs)
                 return NULL;
 
         exynos_encoder = kzalloc(sizeof(*exynos_encoder), GFP_KERNEL);
         if (!exynos_encoder)
                 return NULL;
 
-        exynos_encoder->dpms = DRM_MODE_DPMS_OFF;
-        exynos_encoder->manager = manager;
+        exynos_encoder->display = display;
         encoder = &exynos_encoder->drm_encoder;
         encoder->possible_crtcs = possible_crtcs;
 
@@ -344,149 +191,7 @@ exynos_drm_encoder_create(struct drm_device *dev,
         return encoder;
 }
 
-struct exynos_drm_manager *exynos_drm_get_manager(struct drm_encoder *encoder)
-{
-        return to_exynos_encoder(encoder)->manager;
-}
-
-void exynos_drm_fn_encoder(struct drm_crtc *crtc, void *data,
-                            void (*fn)(struct drm_encoder *, void *))
-{
-        struct drm_device *dev = crtc->dev;
-        struct drm_encoder *encoder;
-        struct exynos_drm_private *private = dev->dev_private;
-        struct exynos_drm_manager *manager;
-
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-                /*
-                 * if crtc is detached from encoder, check pipe,
-                 * otherwise check crtc attached to encoder
-                 */
-                if (!encoder->crtc) {
-                        manager = to_exynos_encoder(encoder)->manager;
-                        if (manager->pipe < 0 ||
-                                        private->crtc[manager->pipe] != crtc)
-                                continue;
-                } else {
-                        if (encoder->crtc != crtc)
-                                continue;
-                }
-
-                fn(encoder, data);
-        }
-}
-
-void exynos_drm_enable_vblank(struct drm_encoder *encoder, void *data)
-{
-        struct exynos_drm_manager *manager =
-                to_exynos_encoder(encoder)->manager;
-        struct exynos_drm_manager_ops *manager_ops = manager->ops;
-        int crtc = *(int *)data;
-
-        if (manager->pipe != crtc)
-                return;
-
-        if (manager_ops->enable_vblank)
-                manager_ops->enable_vblank(manager->dev);
-}
-
-void exynos_drm_disable_vblank(struct drm_encoder *encoder, void *data)
-{
-        struct exynos_drm_manager *manager =
-                to_exynos_encoder(encoder)->manager;
-        struct exynos_drm_manager_ops *manager_ops = manager->ops;
-        int crtc = *(int *)data;
-
-        if (manager->pipe != crtc)
-                return;
-
-        if (manager_ops->disable_vblank)
-                manager_ops->disable_vblank(manager->dev);
-}
-
-void exynos_drm_encoder_crtc_dpms(struct drm_encoder *encoder, void *data)
-{
-        struct exynos_drm_encoder *exynos_encoder = to_exynos_encoder(encoder);
-        struct exynos_drm_manager *manager = exynos_encoder->manager;
-        struct exynos_drm_manager_ops *manager_ops = manager->ops;
-        int mode = *(int *)data;
-
-        if (manager_ops && manager_ops->dpms)
-                manager_ops->dpms(manager->dev, mode);
-
-        /*
-         * if this condition is ok then it means that the crtc is already
-         * detached from encoder and last function for detaching is properly
-         * done, so clear pipe from manager to prevent repeated call.
-         */
-        if (mode > DRM_MODE_DPMS_ON) {
-                if (!encoder->crtc)
-                        manager->pipe = -1;
-        }
-}
-
-void exynos_drm_encoder_crtc_pipe(struct drm_encoder *encoder, void *data)
-{
-        struct exynos_drm_manager *manager =
-                to_exynos_encoder(encoder)->manager;
-        int pipe = *(int *)data;
-
-        /*
-         * when crtc is detached from encoder, this pipe is used
-         * to select manager operation
-         */
-        manager->pipe = pipe;
-}
-
-void exynos_drm_encoder_plane_mode_set(struct drm_encoder *encoder, void *data)
-{
-        struct exynos_drm_manager *manager =
-                to_exynos_encoder(encoder)->manager;
-        struct exynos_drm_overlay_ops *overlay_ops = manager->overlay_ops;
-        struct exynos_drm_overlay *overlay = data;
-
-        if (overlay_ops && overlay_ops->mode_set)
-                overlay_ops->mode_set(manager->dev, overlay);
-}
-
-void exynos_drm_encoder_plane_commit(struct drm_encoder *encoder, void *data)
+struct exynos_drm_display *exynos_drm_get_display(struct drm_encoder *encoder)
 {
-        struct exynos_drm_manager *manager =
-                to_exynos_encoder(encoder)->manager;
-        struct exynos_drm_overlay_ops *overlay_ops = manager->overlay_ops;
-        int zpos = DEFAULT_ZPOS;
-
-        if (data)
-                zpos = *(int *)data;
-
-        if (overlay_ops && overlay_ops->commit)
-                overlay_ops->commit(manager->dev, zpos);
-}
-
-void exynos_drm_encoder_plane_enable(struct drm_encoder *encoder, void *data)
-{
-        struct exynos_drm_manager *manager =
-                to_exynos_encoder(encoder)->manager;
-        struct exynos_drm_overlay_ops *overlay_ops = manager->overlay_ops;
-        int zpos = DEFAULT_ZPOS;
-
-        if (data)
-                zpos = *(int *)data;
-
-        if (overlay_ops && overlay_ops->enable)
-                overlay_ops->enable(manager->dev, zpos);
-}
-
-void exynos_drm_encoder_plane_disable(struct drm_encoder *encoder, void *data)
-{
-        struct exynos_drm_manager *manager =
-                to_exynos_encoder(encoder)->manager;
-        struct exynos_drm_overlay_ops *overlay_ops = manager->overlay_ops;
-        int zpos = DEFAULT_ZPOS;
-
-        if (data)
-                zpos = *(int *)data;
-
-        if (overlay_ops && overlay_ops->disable)
-                overlay_ops->disable(manager->dev, zpos);
+        return to_exynos_encoder(encoder)->display;
 }
diff --git a/drivers/gpu/drm/exynos/exynos_drm_encoder.h b/drivers/gpu/drm/exynos/exynos_drm_encoder.h
index 89e2fb0770af..b7a1620a7e79 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_encoder.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_encoder.h
@@ -18,20 +18,8 @@ struct exynos_drm_manager;
 
 void exynos_drm_encoder_setup(struct drm_device *dev);
 struct drm_encoder *exynos_drm_encoder_create(struct drm_device *dev,
-                                               struct exynos_drm_manager *mgr,
-                                               unsigned int possible_crtcs);
-struct exynos_drm_manager *
-exynos_drm_get_manager(struct drm_encoder *encoder);
-void exynos_drm_fn_encoder(struct drm_crtc *crtc, void *data,
-                            void (*fn)(struct drm_encoder *, void *));
-void exynos_drm_enable_vblank(struct drm_encoder *encoder, void *data);
-void exynos_drm_disable_vblank(struct drm_encoder *encoder, void *data);
-void exynos_drm_encoder_crtc_dpms(struct drm_encoder *encoder, void *data);
-void exynos_drm_encoder_crtc_pipe(struct drm_encoder *encoder, void *data);
-void exynos_drm_encoder_plane_mode_set(struct drm_encoder *encoder, void *data);
-void exynos_drm_encoder_plane_commit(struct drm_encoder *encoder, void *data);
-void exynos_drm_encoder_plane_enable(struct drm_encoder *encoder, void *data);
-void exynos_drm_encoder_plane_disable(struct drm_encoder *encoder, void *data);
-void exynos_drm_encoder_complete_scanout(struct drm_framebuffer *fb);
+                        struct exynos_drm_display *mgr,
+                        unsigned long possible_crtcs);
+struct exynos_drm_display *exynos_drm_get_display(struct drm_encoder *encoder);
 
 #endif
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fb.c b/drivers/gpu/drm/exynos/exynos_drm_fb.c
index ea39e0ef2ae4..65a22cad7b36 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fb.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fb.c
@@ -20,9 +20,10 @@
 
 #include "exynos_drm_drv.h"
 #include "exynos_drm_fb.h"
+#include "exynos_drm_fbdev.h"
 #include "exynos_drm_gem.h"
 #include "exynos_drm_iommu.h"
-#include "exynos_drm_encoder.h"
+#include "exynos_drm_crtc.h"
 
 #define to_exynos_fb(x) container_of(x, struct exynos_drm_fb, fb)
 
@@ -71,7 +72,7 @@ static void exynos_drm_fb_destroy(struct drm_framebuffer *fb)
         unsigned int i;
 
         /* make sure that overlay data are updated before relesing fb. */
-        exynos_drm_encoder_complete_scanout(fb);
+        exynos_drm_crtc_complete_scanout(fb);
 
         drm_framebuffer_cleanup(fb);
 
@@ -300,6 +301,8 @@ static void exynos_drm_output_poll_changed(struct drm_device *dev)
 
         if (fb_helper)
                 drm_fb_helper_hotplug_event(fb_helper);
+        else
+                exynos_drm_fbdev_init(dev);
 }
 
 static const struct drm_mode_config_funcs exynos_drm_mode_config_funcs = {
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
index e7c2f2d07f19..addbf7536da4 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
@@ -90,7 +90,7 @@ static int exynos_drm_fbdev_update(struct drm_fb_helper *helper,
         /* RGB formats use only one buffer */
         buffer = exynos_drm_fb_buffer(fb, 0);
         if (!buffer) {
-                DRM_LOG_KMS("buffer is null.\n");
+                DRM_DEBUG_KMS("buffer is null.\n");
                 return -EFAULT;
         }
 
@@ -237,6 +237,24 @@ static struct drm_fb_helper_funcs exynos_drm_fb_helper_funcs = {
         .fb_probe =     exynos_drm_fbdev_create,
 };
 
+bool exynos_drm_fbdev_is_anything_connected(struct drm_device *dev)
+{
+        struct drm_connector *connector;
+        bool ret = false;
+
+        mutex_lock(&dev->mode_config.mutex);
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                if (connector->status != connector_status_connected)
+                        continue;
+
+                ret = true;
+                break;
+        }
+        mutex_unlock(&dev->mode_config.mutex);
+
+        return ret;
+}
+
 int exynos_drm_fbdev_init(struct drm_device *dev)
 {
         struct exynos_drm_fbdev *fbdev;
@@ -248,6 +266,9 @@ int exynos_drm_fbdev_init(struct drm_device *dev)
         if (!dev->mode_config.num_crtc || !dev->mode_config.num_connector)
                 return 0;
 
+        if (!exynos_drm_fbdev_is_anything_connected(dev))
+                return 0;
+
         fbdev = kzalloc(sizeof(*fbdev), GFP_KERNEL);
         if (!fbdev)
                 return -ENOMEM;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fimd.c b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
index a20440ce32e6..40fd6ccfcd6f 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fimd.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fimd.c
@@ -62,7 +62,7 @@
 /* FIMD has totally five hardware windows. */
 #define WINDOWS_NR      5
 
-#define get_fimd_context(dev)   platform_get_drvdata(to_platform_device(dev))
+#define get_fimd_manager(mgr)   platform_get_drvdata(to_platform_device(dev))
 
 struct fimd_driver_data {
         unsigned int timing_base;
@@ -105,20 +105,18 @@ struct fimd_win_data {
 };
 
 struct fimd_context {
-        struct exynos_drm_subdrv        subdrv;
-        int                             irq;
-        struct drm_crtc                 *crtc;
+        struct device                   *dev;
+        struct drm_device               *drm_dev;
         struct clk                      *bus_clk;
         struct clk                      *lcd_clk;
         void __iomem                    *regs;
+        struct drm_display_mode         mode;
         struct fimd_win_data            win_data[WINDOWS_NR];
-        unsigned int                    clkdiv;
         unsigned int                    default_win;
         unsigned long                   irq_flags;
-        u32                             vidcon0;
         u32                             vidcon1;
         bool                            suspended;
-        struct mutex                    lock;
+        int                             pipe;
         wait_queue_head_t               wait_vsync_queue;
         atomic_t                        wait_vsync_event;
 
@@ -145,153 +143,147 @@ static inline struct fimd_driver_data *drm_fimd_get_driver_data(
         return (struct fimd_driver_data *)of_id->data;
 }
 
-static bool fimd_display_is_connected(struct device *dev)
+static int fimd_mgr_initialize(struct exynos_drm_manager *mgr,
+                        struct drm_device *drm_dev, int pipe)
 {
-        /* TODO. */
+        struct fimd_context *ctx = mgr->ctx;
 
-        return true;
-}
+        ctx->drm_dev = drm_dev;
+        ctx->pipe = pipe;
 
-static void *fimd_get_panel(struct device *dev)
-{
-        struct fimd_context *ctx = get_fimd_context(dev);
+        /*
+         * enable drm irq mode.
+         * - with irq_enabled = true, we can use the vblank feature.
+         *
+         * P.S. note that we wouldn't use drm irq handler but
+         *      just specific driver own one instead because
+         *      drm framework supports only one irq handler.
+         */
+        drm_dev->irq_enabled = true;
 
-        return &ctx->panel;
-}
+        /*
+         * with vblank_disable_allowed = true, vblank interrupt will be disabled
+         * by drm timer once a current process gives up ownership of
+         * vblank event.(after drm_vblank_put function is called)
+         */
+        drm_dev->vblank_disable_allowed = true;
 
-static int fimd_check_mode(struct device *dev, struct drm_display_mode *mode)
-{
-        /* TODO. */
+        /* attach this sub driver to iommu mapping if supported. */
+        if (is_drm_iommu_supported(ctx->drm_dev))
+                drm_iommu_attach_device(ctx->drm_dev, ctx->dev);
 
         return 0;
 }
 
-static int fimd_display_power_on(struct device *dev, int mode)
+static void fimd_mgr_remove(struct exynos_drm_manager *mgr)
 {
-        /* TODO */
+        struct fimd_context *ctx = mgr->ctx;
 
-        return 0;
+        /* detach this sub driver from iommu mapping if supported. */
+        if (is_drm_iommu_supported(ctx->drm_dev))
+                drm_iommu_detach_device(ctx->drm_dev, ctx->dev);
 }
 
-static struct exynos_drm_display_ops fimd_display_ops = {
-        .type = EXYNOS_DISPLAY_TYPE_LCD,
-        .is_connected = fimd_display_is_connected,
-        .get_panel = fimd_get_panel,
-        .check_mode = fimd_check_mode,
-        .power_on = fimd_display_power_on,
-};
-
-static void fimd_dpms(struct device *subdrv_dev, int mode)
+static u32 fimd_calc_clkdiv(struct fimd_context *ctx,
+                const struct drm_display_mode *mode)
 {
-        struct fimd_context *ctx = get_fimd_context(subdrv_dev);
+        unsigned long ideal_clk = mode->htotal * mode->vtotal * mode->vrefresh;
+        u32 clkdiv;
 
-        DRM_DEBUG_KMS("%d\n", mode);
+        /* Find the clock divider value that gets us closest to ideal_clk */
+        clkdiv = DIV_ROUND_UP(clk_get_rate(ctx->lcd_clk), ideal_clk);
 
-        mutex_lock(&ctx->lock);
+        return (clkdiv < 0x100) ? clkdiv : 0xff;
+}
 
-        switch (mode) {
-        case DRM_MODE_DPMS_ON:
-                /*
-                 * enable fimd hardware only if suspended status.
-                 *
-                 * P.S. fimd_dpms function would be called at booting time so
-                 * clk_enable could be called double time.
-                 */
-                if (ctx->suspended)
-                        pm_runtime_get_sync(subdrv_dev);
-                break;
-        case DRM_MODE_DPMS_STANDBY:
-        case DRM_MODE_DPMS_SUSPEND:
-        case DRM_MODE_DPMS_OFF:
-                if (!ctx->suspended)
-                        pm_runtime_put_sync(subdrv_dev);
-                break;
-        default:
-                DRM_DEBUG_KMS("unspecified mode %d\n", mode);
-                break;
-        }
+static bool fimd_mode_fixup(struct exynos_drm_manager *mgr,
+                const struct drm_display_mode *mode,
+                struct drm_display_mode *adjusted_mode)
+{
+        if (adjusted_mode->vrefresh == 0)
+                adjusted_mode->vrefresh = FIMD_DEFAULT_FRAMERATE;
 
-        mutex_unlock(&ctx->lock);
+        return true;
 }
 
-static void fimd_apply(struct device *subdrv_dev)
+static void fimd_mode_set(struct exynos_drm_manager *mgr,
+                const struct drm_display_mode *in_mode)
 {
-        struct fimd_context *ctx = get_fimd_context(subdrv_dev);
-        struct exynos_drm_manager *mgr = ctx->subdrv.manager;
-        struct exynos_drm_manager_ops *mgr_ops = mgr->ops;
-        struct exynos_drm_overlay_ops *ovl_ops = mgr->overlay_ops;
-        struct fimd_win_data *win_data;
-        int i;
-
-        for (i = 0; i < WINDOWS_NR; i++) {
-                win_data = &ctx->win_data[i];
-                if (win_data->enabled && (ovl_ops && ovl_ops->commit))
-                        ovl_ops->commit(subdrv_dev, i);
-        }
+        struct fimd_context *ctx = mgr->ctx;
 
-        if (mgr_ops && mgr_ops->commit)
-                mgr_ops->commit(subdrv_dev);
+        drm_mode_copy(&ctx->mode, in_mode);
 }
 
-static void fimd_commit(struct device *dev)
+static void fimd_commit(struct exynos_drm_manager *mgr)
 {
-        struct fimd_context *ctx = get_fimd_context(dev);
-        struct exynos_drm_panel_info *panel = &ctx->panel;
-        struct videomode *vm = &panel->vm;
+        struct fimd_context *ctx = mgr->ctx;
+        struct drm_display_mode *mode = &ctx->mode;
         struct fimd_driver_data *driver_data;
-        u32 val;
+        u32 val, clkdiv, vidcon1;
+        int vsync_len, vbpd, vfpd, hsync_len, hbpd, hfpd;
 
         driver_data = ctx->driver_data;
         if (ctx->suspended)
                 return;
 
-        /* setup polarity values from machine code. */
-        writel(ctx->vidcon1, ctx->regs + driver_data->timing_base + VIDCON1);
+        /* nothing to do if we haven't set the mode yet */
+        if (mode->htotal == 0 || mode->vtotal == 0)
+                return;
+
+        /* setup polarity values */
+        vidcon1 = ctx->vidcon1;
+        if (mode->flags & DRM_MODE_FLAG_NVSYNC)
+                vidcon1 |= VIDCON1_INV_VSYNC;
+        if (mode->flags & DRM_MODE_FLAG_NHSYNC)
+                vidcon1 |= VIDCON1_INV_HSYNC;
+        writel(vidcon1, ctx->regs + driver_data->timing_base + VIDCON1);
 
         /* setup vertical timing values. */
-        val = VIDTCON0_VBPD(vm->vback_porch - 1) |
-               VIDTCON0_VFPD(vm->vfront_porch - 1) |
-               VIDTCON0_VSPW(vm->vsync_len - 1);
+        vsync_len = mode->crtc_vsync_end - mode->crtc_vsync_start;
+        vbpd = mode->crtc_vtotal - mode->crtc_vsync_end;
+        vfpd = mode->crtc_vsync_start - mode->crtc_vdisplay;
+
+        val = VIDTCON0_VBPD(vbpd - 1) |
+                VIDTCON0_VFPD(vfpd - 1) |
+                VIDTCON0_VSPW(vsync_len - 1);
         writel(val, ctx->regs + driver_data->timing_base + VIDTCON0);
 
         /* setup horizontal timing values.  */
-        val = VIDTCON1_HBPD(vm->hback_porch - 1) |
-               VIDTCON1_HFPD(vm->hfront_porch - 1) |
-               VIDTCON1_HSPW(vm->hsync_len - 1);
+        hsync_len = mode->crtc_hsync_end - mode->crtc_hsync_start;
+        hbpd = mode->crtc_htotal - mode->crtc_hsync_end;
+        hfpd = mode->crtc_hsync_start - mode->crtc_hdisplay;
+
+        val = VIDTCON1_HBPD(hbpd - 1) |
+                VIDTCON1_HFPD(hfpd - 1) |
+                VIDTCON1_HSPW(hsync_len - 1);
         writel(val, ctx->regs + driver_data->timing_base + VIDTCON1);
 
         /* setup horizontal and vertical display size. */
-        val = VIDTCON2_LINEVAL(vm->vactive - 1) |
-               VIDTCON2_HOZVAL(vm->hactive - 1) |
-               VIDTCON2_LINEVAL_E(vm->vactive - 1) |
-               VIDTCON2_HOZVAL_E(vm->hactive - 1);
+        val = VIDTCON2_LINEVAL(mode->vdisplay - 1) |
+               VIDTCON2_HOZVAL(mode->hdisplay - 1) |
+               VIDTCON2_LINEVAL_E(mode->vdisplay - 1) |
+               VIDTCON2_HOZVAL_E(mode->hdisplay - 1);
         writel(val, ctx->regs + driver_data->timing_base + VIDTCON2);
 
-        /* setup clock source, clock divider, enable dma. */
-        val = ctx->vidcon0;
-        val &= ~(VIDCON0_CLKVAL_F_MASK | VIDCON0_CLKDIR);
-
-        if (ctx->driver_data->has_clksel) {
-                val &= ~VIDCON0_CLKSEL_MASK;
-                val |= VIDCON0_CLKSEL_LCD;
-        }
-
-        if (ctx->clkdiv > 1)
-                val |= VIDCON0_CLKVAL_F(ctx->clkdiv - 1) | VIDCON0_CLKDIR;
-        else
-                val &= ~VIDCON0_CLKDIR; /* 1:1 clock */
-
         /*
          * fields of register with prefix '_F' would be updated
          * at vsync(same as dma start)
          */
-        val |= VIDCON0_ENVID | VIDCON0_ENVID_F;
+        val = VIDCON0_ENVID | VIDCON0_ENVID_F;
+
+        if (ctx->driver_data->has_clksel)
+                val |= VIDCON0_CLKSEL_LCD;
+
+        clkdiv = fimd_calc_clkdiv(ctx, mode);
+        if (clkdiv > 1)
+                val |= VIDCON0_CLKVAL_F(clkdiv - 1) | VIDCON0_CLKDIR;
+
         writel(val, ctx->regs + VIDCON0);
 }
 
-static int fimd_enable_vblank(struct device *dev)
+static int fimd_enable_vblank(struct exynos_drm_manager *mgr)
 {
-        struct fimd_context *ctx = get_fimd_context(dev);
+        struct fimd_context *ctx = mgr->ctx;
         u32 val;
 
         if (ctx->suspended)
@@ -314,9 +306,9 @@ static int fimd_enable_vblank(struct device *dev)
         return 0;
 }
 
-static void fimd_disable_vblank(struct device *dev)
+static void fimd_disable_vblank(struct exynos_drm_manager *mgr)
 {
-        struct fimd_context *ctx = get_fimd_context(dev);
+        struct fimd_context *ctx = mgr->ctx;
         u32 val;
 
         if (ctx->suspended)
@@ -332,9 +324,9 @@ static void fimd_disable_vblank(struct device *dev)
         }
 }
 
-static void fimd_wait_for_vblank(struct device *dev)
+static void fimd_wait_for_vblank(struct exynos_drm_manager *mgr)
 {
-        struct fimd_context *ctx = get_fimd_context(dev);
+        struct fimd_context *ctx = mgr->ctx;
 
         if (ctx->suspended)
                 return;
@@ -351,25 +343,16 @@ static void fimd_wait_for_vblank(struct device *dev)
                 DRM_DEBUG_KMS("vblank wait timed out.\n");
 }
 
-static struct exynos_drm_manager_ops fimd_manager_ops = {
-        .dpms = fimd_dpms,
-        .apply = fimd_apply,
-        .commit = fimd_commit,
-        .enable_vblank = fimd_enable_vblank,
-        .disable_vblank = fimd_disable_vblank,
-        .wait_for_vblank = fimd_wait_for_vblank,
-};
-
-static void fimd_win_mode_set(struct device *dev,
-                              struct exynos_drm_overlay *overlay)
+static void fimd_win_mode_set(struct exynos_drm_manager *mgr,
+                        struct exynos_drm_overlay *overlay)
 {
-        struct fimd_context *ctx = get_fimd_context(dev);
+        struct fimd_context *ctx = mgr->ctx;
         struct fimd_win_data *win_data;
         int win;
         unsigned long offset;
 
         if (!overlay) {
-                dev_err(dev, "overlay is NULL\n");
+                DRM_ERROR("overlay is NULL\n");
                 return;
         }
 
@@ -409,9 +392,8 @@ static void fimd_win_mode_set(struct device *dev,
                         overlay->fb_width, overlay->crtc_width);
 }
 
-static void fimd_win_set_pixfmt(struct device *dev, unsigned int win)
+static void fimd_win_set_pixfmt(struct fimd_context *ctx, unsigned int win)
 {
-        struct fimd_context *ctx = get_fimd_context(dev);
         struct fimd_win_data *win_data = &ctx->win_data[win];
         unsigned long val;
 
@@ -467,9 +449,8 @@ static void fimd_win_set_pixfmt(struct device *dev, unsigned int win)
         writel(val, ctx->regs + WINCON(win));
 }
 
-static void fimd_win_set_colkey(struct device *dev, unsigned int win)
+static void fimd_win_set_colkey(struct fimd_context *ctx, unsigned int win)
 {
-        struct fimd_context *ctx = get_fimd_context(dev);
         unsigned int keycon0 = 0, keycon1 = 0;
 
         keycon0 = ~(WxKEYCON0_KEYBL_EN | WxKEYCON0_KEYEN_F |
@@ -508,9 +489,9 @@ static void fimd_shadow_protect_win(struct fimd_context *ctx,
         writel(val, ctx->regs + reg);
 }
 
-static void fimd_win_commit(struct device *dev, int zpos)
+static void fimd_win_commit(struct exynos_drm_manager *mgr, int zpos)
 {
-        struct fimd_context *ctx = get_fimd_context(dev);
+        struct fimd_context *ctx = mgr->ctx;
         struct fimd_win_data *win_data;
         int win = zpos;
         unsigned long val, alpha, size;
@@ -528,6 +509,12 @@ static void fimd_win_commit(struct device *dev, int zpos)
 
         win_data = &ctx->win_data[win];
 
+        /* If suspended, enable this on resume */
+        if (ctx->suspended) {
+                win_data->resume = true;
+                return;
+        }
+
         /*
          * SHADOWCON/PRTCON register is used for enabling timing.
          *
@@ -605,11 +592,11 @@ static void fimd_win_commit(struct device *dev, int zpos)
                 DRM_DEBUG_KMS("osd size = 0x%x\n", (unsigned int)val);
         }
 
-        fimd_win_set_pixfmt(dev, win);
+        fimd_win_set_pixfmt(ctx, win);
 
         /* hardware window 0 doesn't support color key. */
         if (win != 0)
-                fimd_win_set_colkey(dev, win);
+                fimd_win_set_colkey(ctx, win);
 
         /* wincon */
         val = readl(ctx->regs + WINCON(win));
@@ -628,9 +615,9 @@ static void fimd_win_commit(struct device *dev, int zpos)
         win_data->enabled = true;
 }
 
-static void fimd_win_disable(struct device *dev, int zpos)
+static void fimd_win_disable(struct exynos_drm_manager *mgr, int zpos)
 {
-        struct fimd_context *ctx = get_fimd_context(dev);
+        struct fimd_context *ctx = mgr->ctx;
         struct fimd_win_data *win_data;
         int win = zpos;
         u32 val;
@@ -669,132 +656,6 @@ static void fimd_win_disable(struct device *dev, int zpos)
         win_data->enabled = false;
 }
 
-static struct exynos_drm_overlay_ops fimd_overlay_ops = {
-        .mode_set = fimd_win_mode_set,
-        .commit = fimd_win_commit,
-        .disable = fimd_win_disable,
-};
-
-static struct exynos_drm_manager fimd_manager = {
-        .pipe           = -1,
-        .ops            = &fimd_manager_ops,
-        .overlay_ops    = &fimd_overlay_ops,
-        .display_ops    = &fimd_display_ops,
-};
-
-static irqreturn_t fimd_irq_handler(int irq, void *dev_id)
-{
-        struct fimd_context *ctx = (struct fimd_context *)dev_id;
-        struct exynos_drm_subdrv *subdrv = &ctx->subdrv;
-        struct drm_device *drm_dev = subdrv->drm_dev;
-        struct exynos_drm_manager *manager = subdrv->manager;
-        u32 val;
-
-        val = readl(ctx->regs + VIDINTCON1);
-
-        if (val & VIDINTCON1_INT_FRAME)
-                /* VSYNC interrupt */
-                writel(VIDINTCON1_INT_FRAME, ctx->regs + VIDINTCON1);
-
-        /* check the crtc is detached already from encoder */
-        if (manager->pipe < 0)
-                goto out;
-
-        drm_handle_vblank(drm_dev, manager->pipe);
-        exynos_drm_crtc_finish_pageflip(drm_dev, manager->pipe);
-
-        /* set wait vsync event to zero and wake up queue. */
-        if (atomic_read(&ctx->wait_vsync_event)) {
-                atomic_set(&ctx->wait_vsync_event, 0);
-                wake_up(&ctx->wait_vsync_queue);
-        }
-out:
-        return IRQ_HANDLED;
-}
-
-static int fimd_subdrv_probe(struct drm_device *drm_dev, struct device *dev)
-{
-        /*
-         * enable drm irq mode.
-         * - with irq_enabled = true, we can use the vblank feature.
-         *
-         * P.S. note that we wouldn't use drm irq handler but
-         *      just specific driver own one instead because
-         *      drm framework supports only one irq handler.
-         */
-        drm_dev->irq_enabled = true;
-
-        /*
-         * with vblank_disable_allowed = true, vblank interrupt will be disabled
-         * by drm timer once a current process gives up ownership of
-         * vblank event.(after drm_vblank_put function is called)
-         */
-        drm_dev->vblank_disable_allowed = true;
-
-        /* attach this sub driver to iommu mapping if supported. */
-        if (is_drm_iommu_supported(drm_dev))
-                drm_iommu_attach_device(drm_dev, dev);
-
-        return 0;
-}
-
-static void fimd_subdrv_remove(struct drm_device *drm_dev, struct device *dev)
-{
-        /* detach this sub driver from iommu mapping if supported. */
-        if (is_drm_iommu_supported(drm_dev))
-                drm_iommu_detach_device(drm_dev, dev);
-}
-
-static int fimd_configure_clocks(struct fimd_context *ctx, struct device *dev)
-{
-        struct videomode *vm = &ctx->panel.vm;
-        unsigned long clk;
-
-        ctx->bus_clk = devm_clk_get(dev, "fimd");
-        if (IS_ERR(ctx->bus_clk)) {
-                dev_err(dev, "failed to get bus clock\n");
-                return PTR_ERR(ctx->bus_clk);
-        }
-
-        ctx->lcd_clk = devm_clk_get(dev, "sclk_fimd");
-        if (IS_ERR(ctx->lcd_clk)) {
-                dev_err(dev, "failed to get lcd clock\n");
-                return PTR_ERR(ctx->lcd_clk);
-        }
-
-        clk = clk_get_rate(ctx->lcd_clk);
-        if (clk == 0) {
-                dev_err(dev, "error getting sclk_fimd clock rate\n");
-                return -EINVAL;
-        }
-
-        if (vm->pixelclock == 0) {
-                unsigned long c;
-                c = vm->hactive + vm->hback_porch + vm->hfront_porch +
-                    vm->hsync_len;
-                c *= vm->vactive + vm->vback_porch + vm->vfront_porch +
-                     vm->vsync_len;
-                vm->pixelclock = c * FIMD_DEFAULT_FRAMERATE;
-                if (vm->pixelclock == 0) {
-                        dev_err(dev, "incorrect display timings\n");
-                        return -EINVAL;
-                }
-                dev_warn(dev, "pixel clock recalculated to %luHz (%dHz frame rate)\n",
-                         vm->pixelclock, FIMD_DEFAULT_FRAMERATE);
-        }
-        ctx->clkdiv = DIV_ROUND_UP(clk, vm->pixelclock);
-        if (ctx->clkdiv > 256) {
-                dev_warn(dev, "calculated pixel clock divider too high (%u), lowered to 256\n",
-                         ctx->clkdiv);
-                ctx->clkdiv = 256;
-        }
-        vm->pixelclock = clk / ctx->clkdiv;
-        DRM_DEBUG_KMS("pixel clock = %lu, clkdiv = %d\n", vm->pixelclock,
-                      ctx->clkdiv);
-
-        return 0;
-}
-
 static void fimd_clear_win(struct fimd_context *ctx, int win)
 {
         writel(0, ctx->regs + WINCON(win));
@@ -808,111 +669,190 @@ static void fimd_clear_win(struct fimd_context *ctx, int win)
         fimd_shadow_protect_win(ctx, win, false);
 }
 
-static int fimd_clock(struct fimd_context *ctx, bool enable)
+static void fimd_window_suspend(struct exynos_drm_manager *mgr)
 {
-        if (enable) {
-                int ret;
-
-                ret = clk_prepare_enable(ctx->bus_clk);
-                if (ret < 0)
-                        return ret;
+        struct fimd_context *ctx = mgr->ctx;
+        struct fimd_win_data *win_data;
+        int i;
 
-                ret = clk_prepare_enable(ctx->lcd_clk);
-                if  (ret < 0) {
-                        clk_disable_unprepare(ctx->bus_clk);
-                        return ret;
-                }
-        } else {
-                clk_disable_unprepare(ctx->lcd_clk);
-                clk_disable_unprepare(ctx->bus_clk);
+        for (i = 0; i < WINDOWS_NR; i++) {
+                win_data = &ctx->win_data[i];
+                win_data->resume = win_data->enabled;
+                if (win_data->enabled)
+                        fimd_win_disable(mgr, i);
         }
-
-        return 0;
+        fimd_wait_for_vblank(mgr);
 }
 
-static void fimd_window_suspend(struct device *dev)
+static void fimd_window_resume(struct exynos_drm_manager *mgr)
 {
-        struct fimd_context *ctx = get_fimd_context(dev);
+        struct fimd_context *ctx = mgr->ctx;
         struct fimd_win_data *win_data;
         int i;
 
         for (i = 0; i < WINDOWS_NR; i++) {
                 win_data = &ctx->win_data[i];
-                win_data->resume = win_data->enabled;
-                fimd_win_disable(dev, i);
+                win_data->enabled = win_data->resume;
+                win_data->resume = false;
         }
-        fimd_wait_for_vblank(dev);
 }
 
-static void fimd_window_resume(struct device *dev)
+static void fimd_apply(struct exynos_drm_manager *mgr)
 {
-        struct fimd_context *ctx = get_fimd_context(dev);
+        struct fimd_context *ctx = mgr->ctx;
         struct fimd_win_data *win_data;
         int i;
 
         for (i = 0; i < WINDOWS_NR; i++) {
                 win_data = &ctx->win_data[i];
-                win_data->enabled = win_data->resume;
-                win_data->resume = false;
+                if (win_data->enabled)
+                        fimd_win_commit(mgr, i);
         }
+
+        fimd_commit(mgr);
 }
 
-static int fimd_activate(struct fimd_context *ctx, bool enable)
+static int fimd_poweron(struct exynos_drm_manager *mgr)
 {
-        struct device *dev = ctx->subdrv.dev;
-        if (enable) {
-                int ret;
+        struct fimd_context *ctx = mgr->ctx;
+        int ret;
 
-                ret = fimd_clock(ctx, true);
-                if (ret < 0)
-                        return ret;
+        if (!ctx->suspended)
+                return 0;
 
-                ctx->suspended = false;
+        ctx->suspended = false;
 
-                /* if vblank was enabled status, enable it again. */
-                if (test_and_clear_bit(0, &ctx->irq_flags))
-                        fimd_enable_vblank(dev);
+        pm_runtime_get_sync(ctx->dev);
 
-                fimd_window_resume(dev);
-        } else {
-                fimd_window_suspend(dev);
+        ret = clk_prepare_enable(ctx->bus_clk);
+        if (ret < 0) {
+                DRM_ERROR("Failed to prepare_enable the bus clk [%d]\n", ret);
+                goto bus_clk_err;
+        }
+
+        ret = clk_prepare_enable(ctx->lcd_clk);
+        if  (ret < 0) {
+                DRM_ERROR("Failed to prepare_enable the lcd clk [%d]\n", ret);
+                goto lcd_clk_err;
+        }
 
-                fimd_clock(ctx, false);
-                ctx->suspended = true;
+        /* if vblank was enabled status, enable it again. */
+        if (test_and_clear_bit(0, &ctx->irq_flags)) {
+                ret = fimd_enable_vblank(mgr);
+                if (ret) {
+                        DRM_ERROR("Failed to re-enable vblank [%d]\n", ret);
+                        goto enable_vblank_err;
+                }
         }
 
+        fimd_window_resume(mgr);
+
+        fimd_apply(mgr);
+
         return 0;
+
+enable_vblank_err:
+        clk_disable_unprepare(ctx->lcd_clk);
+lcd_clk_err:
+        clk_disable_unprepare(ctx->bus_clk);
+bus_clk_err:
+        ctx->suspended = true;
+        return ret;
 }
 
-static int fimd_get_platform_data(struct fimd_context *ctx, struct device *dev)
+static int fimd_poweroff(struct exynos_drm_manager *mgr)
 {
-        struct videomode *vm;
-        int ret;
+        struct fimd_context *ctx = mgr->ctx;
 
-        vm = &ctx->panel.vm;
-        ret = of_get_videomode(dev->of_node, vm, OF_USE_NATIVE_MODE);
-        if (ret) {
-                DRM_ERROR("failed: of_get_videomode() : %d\n", ret);
-                return ret;
-        }
+        if (ctx->suspended)
+                return 0;
 
-        if (vm->flags & DISPLAY_FLAGS_VSYNC_LOW)
-                ctx->vidcon1 |= VIDCON1_INV_VSYNC;
-        if (vm->flags & DISPLAY_FLAGS_HSYNC_LOW)
-                ctx->vidcon1 |= VIDCON1_INV_HSYNC;
-        if (vm->flags & DISPLAY_FLAGS_DE_LOW)
-                ctx->vidcon1 |= VIDCON1_INV_VDEN;
-        if (vm->flags & DISPLAY_FLAGS_PIXDATA_NEGEDGE)
-                ctx->vidcon1 |= VIDCON1_INV_VCLK;
+        /*
+         * We need to make sure that all windows are disabled before we
+         * suspend that connector. Otherwise we might try to scan from
+         * a destroyed buffer later.
+         */
+        fimd_window_suspend(mgr);
 
+        clk_disable_unprepare(ctx->lcd_clk);
+        clk_disable_unprepare(ctx->bus_clk);
+
+        pm_runtime_put_sync(ctx->dev);
+
+        ctx->suspended = true;
         return 0;
 }
 
+static void fimd_dpms(struct exynos_drm_manager *mgr, int mode)
+{
+        DRM_DEBUG_KMS("%s, %d\n", __FILE__, mode);
+
+        switch (mode) {
+        case DRM_MODE_DPMS_ON:
+                fimd_poweron(mgr);
+                break;
+        case DRM_MODE_DPMS_STANDBY:
+        case DRM_MODE_DPMS_SUSPEND:
+        case DRM_MODE_DPMS_OFF:
+                fimd_poweroff(mgr);
+                break;
+        default:
+                DRM_DEBUG_KMS("unspecified mode %d\n", mode);
+                break;
+        }
+}
+
+static struct exynos_drm_manager_ops fimd_manager_ops = {
+        .initialize = fimd_mgr_initialize,
+        .remove = fimd_mgr_remove,
+        .dpms = fimd_dpms,
+        .mode_fixup = fimd_mode_fixup,
+        .mode_set = fimd_mode_set,
+        .commit = fimd_commit,
+        .enable_vblank = fimd_enable_vblank,
+        .disable_vblank = fimd_disable_vblank,
+        .wait_for_vblank = fimd_wait_for_vblank,
+        .win_mode_set = fimd_win_mode_set,
+        .win_commit = fimd_win_commit,
+        .win_disable = fimd_win_disable,
+};
+
+static struct exynos_drm_manager fimd_manager = {
+        .type = EXYNOS_DISPLAY_TYPE_LCD,
+        .ops = &fimd_manager_ops,
+};
+
+static irqreturn_t fimd_irq_handler(int irq, void *dev_id)
+{
+        struct fimd_context *ctx = (struct fimd_context *)dev_id;
+        u32 val;
+
+        val = readl(ctx->regs + VIDINTCON1);
+
+        if (val & VIDINTCON1_INT_FRAME)
+                /* VSYNC interrupt */
+                writel(VIDINTCON1_INT_FRAME, ctx->regs + VIDINTCON1);
+
+        /* check the crtc is detached already from encoder */
+        if (ctx->pipe < 0 || !ctx->drm_dev)
+                goto out;
+
+        drm_handle_vblank(ctx->drm_dev, ctx->pipe);
+        exynos_drm_crtc_finish_pageflip(ctx->drm_dev, ctx->pipe);
+
+        /* set wait vsync event to zero and wake up queue. */
+        if (atomic_read(&ctx->wait_vsync_event)) {
+                atomic_set(&ctx->wait_vsync_event, 0);
+                wake_up(&ctx->wait_vsync_queue);
+        }
+out:
+        return IRQ_HANDLED;
+}
+
 static int fimd_probe(struct platform_device *pdev)
 {
         struct device *dev = &pdev->dev;
         struct fimd_context *ctx;
-        struct exynos_drm_subdrv *subdrv;
         struct resource *res;
         int win;
         int ret = -EINVAL;
@@ -924,13 +864,25 @@ static int fimd_probe(struct platform_device *pdev)
         if (!ctx)
                 return -ENOMEM;
 
-        ret = fimd_get_platform_data(ctx, dev);
-        if (ret)
-                return ret;
+        ctx->dev = dev;
+        ctx->suspended = true;
 
-        ret = fimd_configure_clocks(ctx, dev);
-        if (ret)
-                return ret;
+        if (of_property_read_bool(dev->of_node, "samsung,invert-vden"))
+                ctx->vidcon1 |= VIDCON1_INV_VDEN;
+        if (of_property_read_bool(dev->of_node, "samsung,invert-vclk"))
+                ctx->vidcon1 |= VIDCON1_INV_VCLK;
+
+        ctx->bus_clk = devm_clk_get(dev, "fimd");
+        if (IS_ERR(ctx->bus_clk)) {
+                dev_err(dev, "failed to get bus clock\n");
+                return PTR_ERR(ctx->bus_clk);
+        }
+
+        ctx->lcd_clk = devm_clk_get(dev, "sclk_fimd");
+        if (IS_ERR(ctx->lcd_clk)) {
+                dev_err(dev, "failed to get lcd clock\n");
+                return PTR_ERR(ctx->lcd_clk);
+        }
 
         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 
@@ -944,9 +896,7 @@ static int fimd_probe(struct platform_device *pdev)
                 return -ENXIO;
         }
 
-        ctx->irq = res->start;
-
-        ret = devm_request_irq(dev, ctx->irq, fimd_irq_handler,
+        ret = devm_request_irq(dev, res->start, fimd_irq_handler,
                                                         0, "drm_fimd", ctx);
         if (ret) {
                 dev_err(dev, "irq request failed.\n");
@@ -957,112 +907,35 @@ static int fimd_probe(struct platform_device *pdev)
         init_waitqueue_head(&ctx->wait_vsync_queue);
         atomic_set(&ctx->wait_vsync_event, 0);
 
-        subdrv = &ctx->subdrv;
+        platform_set_drvdata(pdev, &fimd_manager);
 
-        subdrv->dev = dev;
-        subdrv->manager = &fimd_manager;
-        subdrv->probe = fimd_subdrv_probe;
-        subdrv->remove = fimd_subdrv_remove;
+        fimd_manager.ctx = ctx;
+        exynos_drm_manager_register(&fimd_manager);
 
-        mutex_init(&ctx->lock);
-
-        platform_set_drvdata(pdev, ctx);
+        exynos_dpi_probe(ctx->dev);
 
         pm_runtime_enable(dev);
-        pm_runtime_get_sync(dev);
 
         for (win = 0; win < WINDOWS_NR; win++)
                 fimd_clear_win(ctx, win);
 
-        exynos_drm_subdrv_register(subdrv);
-
         return 0;
 }
 
 static int fimd_remove(struct platform_device *pdev)
 {
-        struct device *dev = &pdev->dev;
-        struct fimd_context *ctx = platform_get_drvdata(pdev);
-
-        exynos_drm_subdrv_unregister(&ctx->subdrv);
-
-        if (ctx->suspended)
-                goto out;
-
-        pm_runtime_set_suspended(dev);
-        pm_runtime_put_sync(dev);
-
-out:
-        pm_runtime_disable(dev);
-
-        return 0;
-}
-
-#ifdef CONFIG_PM_SLEEP
-static int fimd_suspend(struct device *dev)
-{
-        struct fimd_context *ctx = get_fimd_context(dev);
+        struct exynos_drm_manager *mgr = platform_get_drvdata(pdev);
 
-        /*
-         * do not use pm_runtime_suspend(). if pm_runtime_suspend() is
-         * called here, an error would be returned by that interface
-         * because the usage_count of pm runtime is more than 1.
-         */
-        if (!pm_runtime_suspended(dev))
-                return fimd_activate(ctx, false);
+        exynos_dpi_remove(&pdev->dev);
 
-        return 0;
-}
+        exynos_drm_manager_unregister(&fimd_manager);
 
-static int fimd_resume(struct device *dev)
-{
-        struct fimd_context *ctx = get_fimd_context(dev);
+        fimd_dpms(mgr, DRM_MODE_DPMS_OFF);
 
-        /*
-         * if entered to sleep when lcd panel was on, the usage_count
-         * of pm runtime would still be 1 so in this case, fimd driver
-         * should be on directly not drawing on pm runtime interface.
-         */
-        if (!pm_runtime_suspended(dev)) {
-                int ret;
-
-                ret = fimd_activate(ctx, true);
-                if (ret < 0)
-                        return ret;
-
-                /*
-                 * in case of dpms on(standby), fimd_apply function will
-                 * be called by encoder's dpms callback to update fimd's
-                 * registers but in case of sleep wakeup, it's not.
-                 * so fimd_apply function should be called at here.
-                 */
-                fimd_apply(dev);
-        }
+        pm_runtime_disable(&pdev->dev);
 
         return 0;
 }
-#endif
-
-#ifdef CONFIG_PM_RUNTIME
-static int fimd_runtime_suspend(struct device *dev)
-{
-        struct fimd_context *ctx = get_fimd_context(dev);
-
-        return fimd_activate(ctx, false);
-}
-
-static int fimd_runtime_resume(struct device *dev)
-{
-        struct fimd_context *ctx = get_fimd_context(dev);
-
-        return fimd_activate(ctx, true);
-}
-#endif
-
-static const struct dev_pm_ops fimd_pm_ops = {
-        SET_SYSTEM_SLEEP_PM_OPS(fimd_suspend, fimd_resume)
-        SET_RUNTIME_PM_OPS(fimd_runtime_suspend, fimd_runtime_resume, NULL)
-};
 
 struct platform_driver fimd_driver = {
         .probe          = fimd_probe,
@@ -1070,7 +943,6 @@ struct platform_driver fimd_driver = {
         .driver         = {
                 .name   = "exynos4-fb",
                 .owner  = THIS_MODULE,
-                .pm     = &fimd_pm_ops,
                 .of_match_table = fimd_driver_dt_match,
         },
 };
diff --git a/drivers/gpu/drm/exynos/exynos_drm_hdmi.c b/drivers/gpu/drm/exynos/exynos_drm_hdmi.c
deleted file mode 100644
index 8548b974bd59..000000000000
--- a/drivers/gpu/drm/exynos/exynos_drm_hdmi.c
+++ /dev/null
@@ -1,439 +0,0 @@
-/*
- * Copyright (C) 2011 Samsung Electronics Co.Ltd
- * Authors:
- *      Inki Dae <inki.dae@samsung.com>
- *      Seung-Woo Kim <sw0312.kim@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- *
- */
-
-#include <drm/drmP.h>
-
-#include <linux/kernel.h>
-#include <linux/wait.h>
-#include <linux/platform_device.h>
-#include <linux/pm_runtime.h>
-
-#include <drm/exynos_drm.h>
-
-#include "exynos_drm_drv.h"
-#include "exynos_drm_hdmi.h"
-
-#define to_context(dev)         platform_get_drvdata(to_platform_device(dev))
-#define to_subdrv(dev)          to_context(dev)
-#define get_ctx_from_subdrv(subdrv)     container_of(subdrv,\
-                                        struct drm_hdmi_context, subdrv);
-
-/* platform device pointer for common drm hdmi device. */
-static struct platform_device *exynos_drm_hdmi_pdev;
-
-/* Common hdmi subdrv needs to access the hdmi and mixer though context.
-* These should be initialied by the repective drivers */
-static struct exynos_drm_hdmi_context *hdmi_ctx;
-static struct exynos_drm_hdmi_context *mixer_ctx;
-
-/* these callback points shoud be set by specific drivers. */
-static struct exynos_hdmi_ops *hdmi_ops;
-static struct exynos_mixer_ops *mixer_ops;
-
-struct drm_hdmi_context {
-        struct exynos_drm_subdrv        subdrv;
-        struct exynos_drm_hdmi_context  *hdmi_ctx;
-        struct exynos_drm_hdmi_context  *mixer_ctx;
-
-        bool    enabled[MIXER_WIN_NR];
-};
-
-int exynos_platform_device_hdmi_register(void)
-{
-        struct platform_device *pdev;
-
-        if (exynos_drm_hdmi_pdev)
-                return -EEXIST;
-
-        pdev = platform_device_register_simple(
-                        "exynos-drm-hdmi", -1, NULL, 0);
-        if (IS_ERR(pdev))
-                return PTR_ERR(pdev);
-
-        exynos_drm_hdmi_pdev = pdev;
-
-        return 0;
-}
-
-void exynos_platform_device_hdmi_unregister(void)
-{
-        if (exynos_drm_hdmi_pdev) {
-                platform_device_unregister(exynos_drm_hdmi_pdev);
-                exynos_drm_hdmi_pdev = NULL;
-        }
-}
-
-void exynos_hdmi_drv_attach(struct exynos_drm_hdmi_context *ctx)
-{
-        if (ctx)
-                hdmi_ctx = ctx;
-}
-
-void exynos_mixer_drv_attach(struct exynos_drm_hdmi_context *ctx)
-{
-        if (ctx)
-                mixer_ctx = ctx;
-}
-
-void exynos_hdmi_ops_register(struct exynos_hdmi_ops *ops)
-{
-        if (ops)
-                hdmi_ops = ops;
-}
-
-void exynos_mixer_ops_register(struct exynos_mixer_ops *ops)
-{
-        if (ops)
-                mixer_ops = ops;
-}
-
-static bool drm_hdmi_is_connected(struct device *dev)
-{
-        struct drm_hdmi_context *ctx = to_context(dev);
-
-        if (hdmi_ops && hdmi_ops->is_connected)
-                return hdmi_ops->is_connected(ctx->hdmi_ctx->ctx);
-
-        return false;
-}
-
-static struct edid *drm_hdmi_get_edid(struct device *dev,
-                        struct drm_connector *connector)
-{
-        struct drm_hdmi_context *ctx = to_context(dev);
-
-        if (hdmi_ops && hdmi_ops->get_edid)
-                return hdmi_ops->get_edid(ctx->hdmi_ctx->ctx, connector);
-
-        return NULL;
-}
-
-static int drm_hdmi_check_mode(struct device *dev,
-                struct drm_display_mode *mode)
-{
-        struct drm_hdmi_context *ctx = to_context(dev);
-        int ret = 0;
-
-        /*
-        * Both, mixer and hdmi should be able to handle the requested mode.
-        * If any of the two fails, return mode as BAD.
-        */
-
-        if (mixer_ops && mixer_ops->check_mode)
-                ret = mixer_ops->check_mode(ctx->mixer_ctx->ctx, mode);
-
-        if (ret)
-                return ret;
-
-        if (hdmi_ops && hdmi_ops->check_mode)
-                return hdmi_ops->check_mode(ctx->hdmi_ctx->ctx, mode);
-
-        return 0;
-}
-
-static int drm_hdmi_power_on(struct device *dev, int mode)
-{
-        struct drm_hdmi_context *ctx = to_context(dev);
-
-        if (hdmi_ops && hdmi_ops->power_on)
-                return hdmi_ops->power_on(ctx->hdmi_ctx->ctx, mode);
-
-        return 0;
-}
-
-static struct exynos_drm_display_ops drm_hdmi_display_ops = {
-        .type = EXYNOS_DISPLAY_TYPE_HDMI,
-        .is_connected = drm_hdmi_is_connected,
-        .get_edid = drm_hdmi_get_edid,
-        .check_mode = drm_hdmi_check_mode,
-        .power_on = drm_hdmi_power_on,
-};
-
-static int drm_hdmi_enable_vblank(struct device *subdrv_dev)
-{
-        struct drm_hdmi_context *ctx = to_context(subdrv_dev);
-        struct exynos_drm_subdrv *subdrv = &ctx->subdrv;
-        struct exynos_drm_manager *manager = subdrv->manager;
-
-        if (mixer_ops && mixer_ops->enable_vblank)
-                return mixer_ops->enable_vblank(ctx->mixer_ctx->ctx,
-                                                manager->pipe);
-
-        return 0;
-}
-
-static void drm_hdmi_disable_vblank(struct device *subdrv_dev)
-{
-        struct drm_hdmi_context *ctx = to_context(subdrv_dev);
-
-        if (mixer_ops && mixer_ops->disable_vblank)
-                return mixer_ops->disable_vblank(ctx->mixer_ctx->ctx);
-}
-
-static void drm_hdmi_wait_for_vblank(struct device *subdrv_dev)
-{
-        struct drm_hdmi_context *ctx = to_context(subdrv_dev);
-
-        if (mixer_ops && mixer_ops->wait_for_vblank)
-                mixer_ops->wait_for_vblank(ctx->mixer_ctx->ctx);
-}
-
-static void drm_hdmi_mode_fixup(struct device *subdrv_dev,
-                                struct drm_connector *connector,
-                                const struct drm_display_mode *mode,
-                                struct drm_display_mode *adjusted_mode)
-{
-        struct drm_display_mode *m;
-        int mode_ok;
-
-        drm_mode_set_crtcinfo(adjusted_mode, 0);
-
-        mode_ok = drm_hdmi_check_mode(subdrv_dev, adjusted_mode);
-
-        /* just return if user desired mode exists. */
-        if (mode_ok == 0)
-                return;
-
-        /*
-         * otherwise, find the most suitable mode among modes and change it
-         * to adjusted_mode.
-         */
-        list_for_each_entry(m, &connector->modes, head) {
-                mode_ok = drm_hdmi_check_mode(subdrv_dev, m);
-
-                if (mode_ok == 0) {
-                        struct drm_mode_object base;
-                        struct list_head head;
-
-                        DRM_INFO("desired mode doesn't exist so\n");
-                        DRM_INFO("use the most suitable mode among modes.\n");
-
-                        DRM_DEBUG_KMS("Adjusted Mode: [%d]x[%d] [%d]Hz\n",
-                                m->hdisplay, m->vdisplay, m->vrefresh);
-
-                        /* preserve display mode header while copying. */
-                        head = adjusted_mode->head;
-                        base = adjusted_mode->base;
-                        memcpy(adjusted_mode, m, sizeof(*m));
-                        adjusted_mode->head = head;
-                        adjusted_mode->base = base;
-                        break;
-                }
-        }
-}
-
-static void drm_hdmi_mode_set(struct device *subdrv_dev, void *mode)
-{
-        struct drm_hdmi_context *ctx = to_context(subdrv_dev);
-
-        if (hdmi_ops && hdmi_ops->mode_set)
-                hdmi_ops->mode_set(ctx->hdmi_ctx->ctx, mode);
-}
-
-static void drm_hdmi_get_max_resol(struct device *subdrv_dev,
-                                unsigned int *width, unsigned int *height)
-{
-        struct drm_hdmi_context *ctx = to_context(subdrv_dev);
-
-        if (hdmi_ops && hdmi_ops->get_max_resol)
-                hdmi_ops->get_max_resol(ctx->hdmi_ctx->ctx, width, height);
-}
-
-static void drm_hdmi_commit(struct device *subdrv_dev)
-{
-        struct drm_hdmi_context *ctx = to_context(subdrv_dev);
-
-        if (hdmi_ops && hdmi_ops->commit)
-                hdmi_ops->commit(ctx->hdmi_ctx->ctx);
-}
-
-static void drm_hdmi_dpms(struct device *subdrv_dev, int mode)
-{
-        struct drm_hdmi_context *ctx = to_context(subdrv_dev);
-
-        if (mixer_ops && mixer_ops->dpms)
-                mixer_ops->dpms(ctx->mixer_ctx->ctx, mode);
-
-        if (hdmi_ops && hdmi_ops->dpms)
-                hdmi_ops->dpms(ctx->hdmi_ctx->ctx, mode);
-}
-
-static void drm_hdmi_apply(struct device *subdrv_dev)
-{
-        struct drm_hdmi_context *ctx = to_context(subdrv_dev);
-        int i;
-
-        for (i = 0; i < MIXER_WIN_NR; i++) {
-                if (!ctx->enabled[i])
-                        continue;
-                if (mixer_ops && mixer_ops->win_commit)
-                        mixer_ops->win_commit(ctx->mixer_ctx->ctx, i);
-        }
-
-        if (hdmi_ops && hdmi_ops->commit)
-                hdmi_ops->commit(ctx->hdmi_ctx->ctx);
-}
-
-static struct exynos_drm_manager_ops drm_hdmi_manager_ops = {
-        .dpms = drm_hdmi_dpms,
-        .apply = drm_hdmi_apply,
-        .enable_vblank = drm_hdmi_enable_vblank,
-        .disable_vblank = drm_hdmi_disable_vblank,
-        .wait_for_vblank = drm_hdmi_wait_for_vblank,
-        .mode_fixup = drm_hdmi_mode_fixup,
-        .mode_set = drm_hdmi_mode_set,
-        .get_max_resol = drm_hdmi_get_max_resol,
-        .commit = drm_hdmi_commit,
-};
-
-static void drm_mixer_mode_set(struct device *subdrv_dev,
-                struct exynos_drm_overlay *overlay)
-{
-        struct drm_hdmi_context *ctx = to_context(subdrv_dev);
-
-        if (mixer_ops && mixer_ops->win_mode_set)
-                mixer_ops->win_mode_set(ctx->mixer_ctx->ctx, overlay);
-}
-
-static void drm_mixer_commit(struct device *subdrv_dev, int zpos)
-{
-        struct drm_hdmi_context *ctx = to_context(subdrv_dev);
-        int win = (zpos == DEFAULT_ZPOS) ? MIXER_DEFAULT_WIN : zpos;
-
-        if (win < 0 || win >= MIXER_WIN_NR) {
-                DRM_ERROR("mixer window[%d] is wrong\n", win);
-                return;
-        }
-
-        if (mixer_ops && mixer_ops->win_commit)
-                mixer_ops->win_commit(ctx->mixer_ctx->ctx, win);
-
-        ctx->enabled[win] = true;
-}
-
-static void drm_mixer_disable(struct device *subdrv_dev, int zpos)
-{
-        struct drm_hdmi_context *ctx = to_context(subdrv_dev);
-        int win = (zpos == DEFAULT_ZPOS) ? MIXER_DEFAULT_WIN : zpos;
-
-        if (win < 0 || win >= MIXER_WIN_NR) {
-                DRM_ERROR("mixer window[%d] is wrong\n", win);
-                return;
-        }
-
-        if (mixer_ops && mixer_ops->win_disable)
-                mixer_ops->win_disable(ctx->mixer_ctx->ctx, win);
-
-        ctx->enabled[win] = false;
-}
-
-static struct exynos_drm_overlay_ops drm_hdmi_overlay_ops = {
-        .mode_set = drm_mixer_mode_set,
-        .commit = drm_mixer_commit,
-        .disable = drm_mixer_disable,
-};
-
-static struct exynos_drm_manager hdmi_manager = {
-        .pipe           = -1,
-        .ops            = &drm_hdmi_manager_ops,
-        .overlay_ops    = &drm_hdmi_overlay_ops,
-        .display_ops    = &drm_hdmi_display_ops,
-};
-
-static int hdmi_subdrv_probe(struct drm_device *drm_dev,
-                struct device *dev)
-{
-        struct exynos_drm_subdrv *subdrv = to_subdrv(dev);
-        struct drm_hdmi_context *ctx;
-
-        if (!hdmi_ctx) {
-                DRM_ERROR("hdmi context not initialized.\n");
-                return -EFAULT;
-        }
-
-        if (!mixer_ctx) {
-                DRM_ERROR("mixer context not initialized.\n");
-                return -EFAULT;
-        }
-
-        ctx = get_ctx_from_subdrv(subdrv);
-
-        if (!ctx) {
-                DRM_ERROR("no drm hdmi context.\n");
-                return -EFAULT;
-        }
-
-        ctx->hdmi_ctx = hdmi_ctx;
-        ctx->mixer_ctx = mixer_ctx;
-
-        ctx->hdmi_ctx->drm_dev = drm_dev;
-        ctx->mixer_ctx->drm_dev = drm_dev;
-
-        if (mixer_ops->iommu_on)
-                mixer_ops->iommu_on(ctx->mixer_ctx->ctx, true);
-
-        return 0;
-}
-
-static void hdmi_subdrv_remove(struct drm_device *drm_dev, struct device *dev)
-{
-        struct drm_hdmi_context *ctx;
-        struct exynos_drm_subdrv *subdrv = to_subdrv(dev);
-
-        ctx = get_ctx_from_subdrv(subdrv);
-
-        if (mixer_ops->iommu_on)
-                mixer_ops->iommu_on(ctx->mixer_ctx->ctx, false);
-}
-
-static int exynos_drm_hdmi_probe(struct platform_device *pdev)
-{
-        struct device *dev = &pdev->dev;
-        struct exynos_drm_subdrv *subdrv;
-        struct drm_hdmi_context *ctx;
-
-        ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
-        if (!ctx)
-                return -ENOMEM;
-
-        subdrv = &ctx->subdrv;
-
-        subdrv->dev = dev;
-        subdrv->manager = &hdmi_manager;
-        subdrv->probe = hdmi_subdrv_probe;
-        subdrv->remove = hdmi_subdrv_remove;
-
-        platform_set_drvdata(pdev, subdrv);
-
-        exynos_drm_subdrv_register(subdrv);
-
-        return 0;
-}
-
-static int exynos_drm_hdmi_remove(struct platform_device *pdev)
-{
-        struct drm_hdmi_context *ctx = platform_get_drvdata(pdev);
-
-        exynos_drm_subdrv_unregister(&ctx->subdrv);
-
-        return 0;
-}
-
-struct platform_driver exynos_drm_common_hdmi_driver = {
-        .probe          = exynos_drm_hdmi_probe,
-        .remove         = exynos_drm_hdmi_remove,
-        .driver         = {
-                .name   = "exynos-drm-hdmi",
-                .owner  = THIS_MODULE,
-        },
-};
diff --git a/drivers/gpu/drm/exynos/exynos_drm_hdmi.h b/drivers/gpu/drm/exynos/exynos_drm_hdmi.h
deleted file mode 100644
index 724cab181976..000000000000
--- a/drivers/gpu/drm/exynos/exynos_drm_hdmi.h
+++ /dev/null
@@ -1,67 +0,0 @@
-/* exynos_drm_hdmi.h
- *
- * Copyright (c) 2011 Samsung Electronics Co., Ltd.
- * Authoer: Inki Dae <inki.dae@samsung.com>
- *
- * This program is free software; you can redistribute  it and/or modify it
- * under  the terms of  the GNU General  Public License as published by the
- * Free Software Foundation;  either version 2 of the  License, or (at your
- * option) any later version.
- */
-
-#ifndef _EXYNOS_DRM_HDMI_H_
-#define _EXYNOS_DRM_HDMI_H_
-
-#define MIXER_WIN_NR            3
-#define MIXER_DEFAULT_WIN       0
-
-/*
- * exynos hdmi common context structure.
- *
- * @drm_dev: pointer to drm_device.
- * @ctx: pointer to the context of specific device driver.
- *      this context should be hdmi_context or mixer_context.
- */
-struct exynos_drm_hdmi_context {
-        struct drm_device       *drm_dev;
-        void                    *ctx;
-};
-
-struct exynos_hdmi_ops {
-        /* display */
-        bool (*is_connected)(void *ctx);
-        struct edid *(*get_edid)(void *ctx,
-                        struct drm_connector *connector);
-        int (*check_mode)(void *ctx, struct drm_display_mode *mode);
-        int (*power_on)(void *ctx, int mode);
-
-        /* manager */
-        void (*mode_set)(void *ctx, struct drm_display_mode *mode);
-        void (*get_max_resol)(void *ctx, unsigned int *width,
-                                unsigned int *height);
-        void (*commit)(void *ctx);
-        void (*dpms)(void *ctx, int mode);
-};
-
-struct exynos_mixer_ops {
-        /* manager */
-        int (*iommu_on)(void *ctx, bool enable);
-        int (*enable_vblank)(void *ctx, int pipe);
-        void (*disable_vblank)(void *ctx);
-        void (*wait_for_vblank)(void *ctx);
-        void (*dpms)(void *ctx, int mode);
-
-        /* overlay */
-        void (*win_mode_set)(void *ctx, struct exynos_drm_overlay *overlay);
-        void (*win_commit)(void *ctx, int zpos);
-        void (*win_disable)(void *ctx, int zpos);
-
-        /* display */
-        int (*check_mode)(void *ctx, struct drm_display_mode *mode);
-};
-
-void exynos_hdmi_drv_attach(struct exynos_drm_hdmi_context *ctx);
-void exynos_mixer_drv_attach(struct exynos_drm_hdmi_context *ctx);
-void exynos_hdmi_ops_register(struct exynos_hdmi_ops *ops);
-void exynos_mixer_ops_register(struct exynos_mixer_ops *ops);
-#endif
diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.c b/drivers/gpu/drm/exynos/exynos_drm_plane.c
index fcb0652e77d0..8371cbd7631d 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_plane.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_plane.c
@@ -13,7 +13,7 @@
 
 #include <drm/exynos_drm.h>
 #include "exynos_drm_drv.h"
-#include "exynos_drm_encoder.h"
+#include "exynos_drm_crtc.h"
 #include "exynos_drm_fb.h"
 #include "exynos_drm_gem.h"
 #include "exynos_drm_plane.h"
@@ -87,7 +87,7 @@ int exynos_plane_mode_set(struct drm_plane *plane, struct drm_crtc *crtc,
                 struct exynos_drm_gem_buf *buffer = exynos_drm_fb_buffer(fb, i);
 
                 if (!buffer) {
-                        DRM_LOG_KMS("buffer is null\n");
+                        DRM_DEBUG_KMS("buffer is null\n");
                         return -EFAULT;
                 }
 
@@ -139,7 +139,7 @@ int exynos_plane_mode_set(struct drm_plane *plane, struct drm_crtc *crtc,
                         overlay->crtc_x, overlay->crtc_y,
                         overlay->crtc_width, overlay->crtc_height);
 
-        exynos_drm_fn_encoder(crtc, overlay, exynos_drm_encoder_plane_mode_set);
+        exynos_drm_crtc_plane_mode_set(crtc, overlay);
 
         return 0;
 }
@@ -149,8 +149,7 @@ void exynos_plane_commit(struct drm_plane *plane)
         struct exynos_plane *exynos_plane = to_exynos_plane(plane);
         struct exynos_drm_overlay *overlay = &exynos_plane->overlay;
 
-        exynos_drm_fn_encoder(plane->crtc, &overlay->zpos,
-                        exynos_drm_encoder_plane_commit);
+        exynos_drm_crtc_plane_commit(plane->crtc, overlay->zpos);
 }
 
 void exynos_plane_dpms(struct drm_plane *plane, int mode)
@@ -162,17 +161,13 @@ void exynos_plane_dpms(struct drm_plane *plane, int mode)
                 if (exynos_plane->enabled)
                         return;
 
-                exynos_drm_fn_encoder(plane->crtc, &overlay->zpos,
-                                exynos_drm_encoder_plane_enable);
-
+                exynos_drm_crtc_plane_enable(plane->crtc, overlay->zpos);
                 exynos_plane->enabled = true;
         } else {
                 if (!exynos_plane->enabled)
                         return;
 
-                exynos_drm_fn_encoder(plane->crtc, &overlay->zpos,
-                                exynos_drm_encoder_plane_disable);
-
+                exynos_drm_crtc_plane_disable(plane->crtc, overlay->zpos);
                 exynos_plane->enabled = false;
         }
 }
@@ -259,7 +254,7 @@ static void exynos_plane_attach_zpos_property(struct drm_plane *plane)
 }
 
 struct drm_plane *exynos_plane_init(struct drm_device *dev,
-                                    unsigned int possible_crtcs, bool priv)
+                                    unsigned long possible_crtcs, bool priv)
 {
         struct exynos_plane *exynos_plane;
         int err;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_plane.h b/drivers/gpu/drm/exynos/exynos_drm_plane.h
index 88312458580d..84d464c90d3d 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_plane.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_plane.h
@@ -17,4 +17,4 @@ int exynos_plane_mode_set(struct drm_plane *plane, struct drm_crtc *crtc,
 void exynos_plane_commit(struct drm_plane *plane);
 void exynos_plane_dpms(struct drm_plane *plane, int mode);
 struct drm_plane *exynos_plane_init(struct drm_device *dev,
-                                    unsigned int possible_crtcs, bool priv);
+                                    unsigned long possible_crtcs, bool priv);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_vidi.c b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
index ddaaedde173d..7afead9c3f30 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_vidi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
@@ -28,7 +28,9 @@
 /* vidi has totally three virtual windows. */
 #define WINDOWS_NR              3
 
-#define get_vidi_context(dev)   platform_get_drvdata(to_platform_device(dev))
+#define get_vidi_mgr(dev)       platform_get_drvdata(to_platform_device(dev))
+#define ctx_from_connector(c)   container_of(c, struct vidi_context, \
+                                        connector)
 
 struct vidi_win_data {
         unsigned int            offset_x;
@@ -45,8 +47,10 @@ struct vidi_win_data {
 };
 
 struct vidi_context {
-        struct exynos_drm_subdrv        subdrv;
+        struct drm_device               *drm_dev;
         struct drm_crtc                 *crtc;
+        struct drm_encoder              *encoder;
+        struct drm_connector            connector;
         struct vidi_win_data            win_data[WINDOWS_NR];
         struct edid                     *raw_edid;
         unsigned int                    clkdiv;
@@ -58,6 +62,7 @@ struct vidi_context {
         bool                            direct_vblank;
         struct work_struct              work;
         struct mutex                    lock;
+        int                             pipe;
 };
 
 static const char fake_edid_info[] = {
@@ -85,126 +90,34 @@ static const char fake_edid_info[] = {
         0x00, 0x00, 0x00, 0x06
 };
 
-static bool vidi_display_is_connected(struct device *dev)
+static void vidi_apply(struct exynos_drm_manager *mgr)
 {
-        struct vidi_context *ctx = get_vidi_context(dev);
-
-        /*
-         * connection request would come from user side
-         * to do hotplug through specific ioctl.
-         */
-        return ctx->connected ? true : false;
-}
-
-static struct edid *vidi_get_edid(struct device *dev,
-                        struct drm_connector *connector)
-{
-        struct vidi_context *ctx = get_vidi_context(dev);
-        struct edid *edid;
-
-        /*
-         * the edid data comes from user side and it would be set
-         * to ctx->raw_edid through specific ioctl.
-         */
-        if (!ctx->raw_edid) {
-                DRM_DEBUG_KMS("raw_edid is null.\n");
-                return ERR_PTR(-EFAULT);
-        }
-
-        edid = drm_edid_duplicate(ctx->raw_edid);
-        if (!edid) {
-                DRM_DEBUG_KMS("failed to allocate edid\n");
-                return ERR_PTR(-ENOMEM);
-        }
-
-        return edid;
-}
-
-static void *vidi_get_panel(struct device *dev)
-{
-        /* TODO. */
-
-        return NULL;
-}
-
-static int vidi_check_mode(struct device *dev, struct drm_display_mode *mode)
-{
-        /* TODO. */
-
-        return 0;
-}
-
-static int vidi_display_power_on(struct device *dev, int mode)
-{
-        /* TODO */
-
-        return 0;
-}
-
-static struct exynos_drm_display_ops vidi_display_ops = {
-        .type = EXYNOS_DISPLAY_TYPE_VIDI,
-        .is_connected = vidi_display_is_connected,
-        .get_edid = vidi_get_edid,
-        .get_panel = vidi_get_panel,
-        .check_mode = vidi_check_mode,
-        .power_on = vidi_display_power_on,
-};
-
-static void vidi_dpms(struct device *subdrv_dev, int mode)
-{
-        struct vidi_context *ctx = get_vidi_context(subdrv_dev);
-
-        DRM_DEBUG_KMS("%d\n", mode);
-
-        mutex_lock(&ctx->lock);
-
-        switch (mode) {
-        case DRM_MODE_DPMS_ON:
-                /* TODO. */
-                break;
-        case DRM_MODE_DPMS_STANDBY:
-        case DRM_MODE_DPMS_SUSPEND:
-        case DRM_MODE_DPMS_OFF:
-                /* TODO. */
-                break;
-        default:
-                DRM_DEBUG_KMS("unspecified mode %d\n", mode);
-                break;
-        }
-
-        mutex_unlock(&ctx->lock);
-}
-
-static void vidi_apply(struct device *subdrv_dev)
-{
-        struct vidi_context *ctx = get_vidi_context(subdrv_dev);
-        struct exynos_drm_manager *mgr = ctx->subdrv.manager;
+        struct vidi_context *ctx = mgr->ctx;
         struct exynos_drm_manager_ops *mgr_ops = mgr->ops;
-        struct exynos_drm_overlay_ops *ovl_ops = mgr->overlay_ops;
         struct vidi_win_data *win_data;
         int i;
 
         for (i = 0; i < WINDOWS_NR; i++) {
                 win_data = &ctx->win_data[i];
-                if (win_data->enabled && (ovl_ops && ovl_ops->commit))
-                        ovl_ops->commit(subdrv_dev, i);
+                if (win_data->enabled && (mgr_ops && mgr_ops->win_commit))
+                        mgr_ops->win_commit(mgr, i);
         }
 
         if (mgr_ops && mgr_ops->commit)
-                mgr_ops->commit(subdrv_dev);
+                mgr_ops->commit(mgr);
 }
 
-static void vidi_commit(struct device *dev)
+static void vidi_commit(struct exynos_drm_manager *mgr)
 {
-        struct vidi_context *ctx = get_vidi_context(dev);
+        struct vidi_context *ctx = mgr->ctx;
 
         if (ctx->suspended)
                 return;
 }
 
-static int vidi_enable_vblank(struct device *dev)
+static int vidi_enable_vblank(struct exynos_drm_manager *mgr)
 {
-        struct vidi_context *ctx = get_vidi_context(dev);
+        struct vidi_context *ctx = mgr->ctx;
 
         if (ctx->suspended)
                 return -EPERM;
@@ -217,16 +130,16 @@ static int vidi_enable_vblank(struct device *dev)
         /*
          * in case of page flip request, vidi_finish_pageflip function
          * will not be called because direct_vblank is true and then
-         * that function will be called by overlay_ops->commit callback
+         * that function will be called by manager_ops->win_commit callback
          */
         schedule_work(&ctx->work);
 
         return 0;
 }
 
-static void vidi_disable_vblank(struct device *dev)
+static void vidi_disable_vblank(struct exynos_drm_manager *mgr)
 {
-        struct vidi_context *ctx = get_vidi_context(dev);
+        struct vidi_context *ctx = mgr->ctx;
 
         if (ctx->suspended)
                 return;
@@ -235,24 +148,16 @@ static void vidi_disable_vblank(struct device *dev)
                 ctx->vblank_on = false;
 }
 
-static struct exynos_drm_manager_ops vidi_manager_ops = {
-        .dpms = vidi_dpms,
-        .apply = vidi_apply,
-        .commit = vidi_commit,
-        .enable_vblank = vidi_enable_vblank,
-        .disable_vblank = vidi_disable_vblank,
-};
-
-static void vidi_win_mode_set(struct device *dev,
-                              struct exynos_drm_overlay *overlay)
+static void vidi_win_mode_set(struct exynos_drm_manager *mgr,
+                        struct exynos_drm_overlay *overlay)
 {
-        struct vidi_context *ctx = get_vidi_context(dev);
+        struct vidi_context *ctx = mgr->ctx;
         struct vidi_win_data *win_data;
         int win;
         unsigned long offset;
 
         if (!overlay) {
-                dev_err(dev, "overlay is NULL\n");
+                DRM_ERROR("overlay is NULL\n");
                 return;
         }
 
@@ -296,9 +201,9 @@ static void vidi_win_mode_set(struct device *dev,
                         overlay->fb_width, overlay->crtc_width);
 }
 
-static void vidi_win_commit(struct device *dev, int zpos)
+static void vidi_win_commit(struct exynos_drm_manager *mgr, int zpos)
 {
-        struct vidi_context *ctx = get_vidi_context(dev);
+        struct vidi_context *ctx = mgr->ctx;
         struct vidi_win_data *win_data;
         int win = zpos;
 
@@ -321,9 +226,9 @@ static void vidi_win_commit(struct device *dev, int zpos)
                 schedule_work(&ctx->work);
 }
 
-static void vidi_win_disable(struct device *dev, int zpos)
+static void vidi_win_disable(struct exynos_drm_manager *mgr, int zpos)
 {
-        struct vidi_context *ctx = get_vidi_context(dev);
+        struct vidi_context *ctx = mgr->ctx;
         struct vidi_win_data *win_data;
         int win = zpos;
 
@@ -339,98 +244,132 @@ static void vidi_win_disable(struct device *dev, int zpos)
         /* TODO. */
 }
 
-static struct exynos_drm_overlay_ops vidi_overlay_ops = {
-        .mode_set = vidi_win_mode_set,
-        .commit = vidi_win_commit,
-        .disable = vidi_win_disable,
-};
+static int vidi_power_on(struct exynos_drm_manager *mgr, bool enable)
+{
+        struct vidi_context *ctx = mgr->ctx;
 
-static struct exynos_drm_manager vidi_manager = {
-        .pipe           = -1,
-        .ops            = &vidi_manager_ops,
-        .overlay_ops    = &vidi_overlay_ops,
-        .display_ops    = &vidi_display_ops,
-};
+        DRM_DEBUG_KMS("%s\n", __FILE__);
 
-static void vidi_fake_vblank_handler(struct work_struct *work)
-{
-        struct vidi_context *ctx = container_of(work, struct vidi_context,
-                                        work);
-        struct exynos_drm_subdrv *subdrv = &ctx->subdrv;
-        struct exynos_drm_manager *manager = subdrv->manager;
+        if (enable != false && enable != true)
+                return -EINVAL;
 
-        if (manager->pipe < 0)
-                return;
+        if (enable) {
+                ctx->suspended = false;
 
-        /* refresh rate is about 50Hz. */
-        usleep_range(16000, 20000);
+                /* if vblank was enabled status, enable it again. */
+                if (test_and_clear_bit(0, &ctx->irq_flags))
+                        vidi_enable_vblank(mgr);
+
+                vidi_apply(mgr);
+        } else {
+                ctx->suspended = true;
+        }
+
+        return 0;
+}
+
+static void vidi_dpms(struct exynos_drm_manager *mgr, int mode)
+{
+        struct vidi_context *ctx = mgr->ctx;
+
+        DRM_DEBUG_KMS("%d\n", mode);
 
         mutex_lock(&ctx->lock);
 
-        if (ctx->direct_vblank) {
-                drm_handle_vblank(subdrv->drm_dev, manager->pipe);
-                ctx->direct_vblank = false;
-                mutex_unlock(&ctx->lock);
-                return;
+        switch (mode) {
+        case DRM_MODE_DPMS_ON:
+                vidi_power_on(mgr, true);
+                break;
+        case DRM_MODE_DPMS_STANDBY:
+        case DRM_MODE_DPMS_SUSPEND:
+        case DRM_MODE_DPMS_OFF:
+                vidi_power_on(mgr, false);
+                break;
+        default:
+                DRM_DEBUG_KMS("unspecified mode %d\n", mode);
+                break;
         }
 
         mutex_unlock(&ctx->lock);
-
-        exynos_drm_crtc_finish_pageflip(subdrv->drm_dev, manager->pipe);
 }
 
-static int vidi_subdrv_probe(struct drm_device *drm_dev, struct device *dev)
+static int vidi_mgr_initialize(struct exynos_drm_manager *mgr,
+                        struct drm_device *drm_dev, int pipe)
 {
+        struct vidi_context *ctx = mgr->ctx;
+
+        DRM_ERROR("vidi initialize ct=%p dev=%p pipe=%d\n", ctx, drm_dev, pipe);
+
+        ctx->drm_dev = drm_dev;
+        ctx->pipe = pipe;
+
         /*
          * enable drm irq mode.
-         * - with irq_enabled = true, we can use the vblank feature.
+         * - with irq_enabled = 1, we can use the vblank feature.
          *
          * P.S. note that we wouldn't use drm irq handler but
          *      just specific driver own one instead because
          *      drm framework supports only one irq handler.
          */
-        drm_dev->irq_enabled = true;
+        drm_dev->irq_enabled = 1;
 
         /*
-         * with vblank_disable_allowed = true, vblank interrupt will be disabled
+         * with vblank_disable_allowed = 1, vblank interrupt will be disabled
          * by drm timer once a current process gives up ownership of
          * vblank event.(after drm_vblank_put function is called)
          */
-        drm_dev->vblank_disable_allowed = true;
+        drm_dev->vblank_disable_allowed = 1;
 
         return 0;
 }
 
-static void vidi_subdrv_remove(struct drm_device *drm_dev, struct device *dev)
-{
-        /* TODO. */
-}
+static struct exynos_drm_manager_ops vidi_manager_ops = {
+        .initialize = vidi_mgr_initialize,
+        .dpms = vidi_dpms,
+        .commit = vidi_commit,
+        .enable_vblank = vidi_enable_vblank,
+        .disable_vblank = vidi_disable_vblank,
+        .win_mode_set = vidi_win_mode_set,
+        .win_commit = vidi_win_commit,
+        .win_disable = vidi_win_disable,
+};
 
-static int vidi_power_on(struct vidi_context *ctx, bool enable)
+static struct exynos_drm_manager vidi_manager = {
+        .type = EXYNOS_DISPLAY_TYPE_VIDI,
+        .ops = &vidi_manager_ops,
+};
+
+static void vidi_fake_vblank_handler(struct work_struct *work)
 {
-        struct exynos_drm_subdrv *subdrv = &ctx->subdrv;
-        struct device *dev = subdrv->dev;
+        struct vidi_context *ctx = container_of(work, struct vidi_context,
+                                        work);
 
-        if (enable) {
-                ctx->suspended = false;
+        if (ctx->pipe < 0)
+                return;
 
-                /* if vblank was enabled status, enable it again. */
-                if (test_and_clear_bit(0, &ctx->irq_flags))
-                        vidi_enable_vblank(dev);
+        /* refresh rate is about 50Hz. */
+        usleep_range(16000, 20000);
 
-                vidi_apply(dev);
-        } else {
-                ctx->suspended = true;
+        mutex_lock(&ctx->lock);
+
+        if (ctx->direct_vblank) {
+                drm_handle_vblank(ctx->drm_dev, ctx->pipe);
+                ctx->direct_vblank = false;
+                mutex_unlock(&ctx->lock);
+                return;
         }
 
-        return 0;
+        mutex_unlock(&ctx->lock);
+
+        exynos_drm_crtc_finish_pageflip(ctx->drm_dev, ctx->pipe);
 }
 
 static int vidi_show_connection(struct device *dev,
                                 struct device_attribute *attr, char *buf)
 {
         int rc;
-        struct vidi_context *ctx = get_vidi_context(dev);
+        struct exynos_drm_manager *mgr = get_vidi_mgr(dev);
+        struct vidi_context *ctx = mgr->ctx;
 
         mutex_lock(&ctx->lock);
 
@@ -445,7 +384,8 @@ static int vidi_store_connection(struct device *dev,
                                 struct device_attribute *attr,
                                 const char *buf, size_t len)
 {
-        struct vidi_context *ctx = get_vidi_context(dev);
+        struct exynos_drm_manager *mgr = get_vidi_mgr(dev);
+        struct vidi_context *ctx = mgr->ctx;
         int ret;
 
         ret = kstrtoint(buf, 0, &ctx->connected);
@@ -467,7 +407,7 @@ static int vidi_store_connection(struct device *dev,
 
         DRM_DEBUG_KMS("requested connection.\n");
 
-        drm_helper_hpd_irq_event(ctx->subdrv.drm_dev);
+        drm_helper_hpd_irq_event(ctx->drm_dev);
 
         return len;
 }
@@ -480,8 +420,7 @@ int vidi_connection_ioctl(struct drm_device *drm_dev, void *data,
 {
         struct vidi_context *ctx = NULL;
         struct drm_encoder *encoder;
-        struct exynos_drm_manager *manager;
-        struct exynos_drm_display_ops *display_ops;
+        struct exynos_drm_display *display;
         struct drm_exynos_vidi_connection *vidi = data;
 
         if (!vidi) {
@@ -496,11 +435,10 @@ int vidi_connection_ioctl(struct drm_device *drm_dev, void *data,
 
         list_for_each_entry(encoder, &drm_dev->mode_config.encoder_list,
                                                                 head) {
-                manager = exynos_drm_get_manager(encoder);
-                display_ops = manager->display_ops;
+                display = exynos_drm_get_display(encoder);
 
-                if (display_ops->type == EXYNOS_DISPLAY_TYPE_VIDI) {
-                        ctx = get_vidi_context(manager->dev);
+                if (display->type == EXYNOS_DISPLAY_TYPE_VIDI) {
+                        ctx = display->ctx;
                         break;
                 }
         }
@@ -539,16 +477,119 @@ int vidi_connection_ioctl(struct drm_device *drm_dev, void *data,
         }
 
         ctx->connected = vidi->connection;
-        drm_helper_hpd_irq_event(ctx->subdrv.drm_dev);
+        drm_helper_hpd_irq_event(ctx->drm_dev);
+
+        return 0;
+}
+
+static enum drm_connector_status vidi_detect(struct drm_connector *connector,
+                        bool force)
+{
+        struct vidi_context *ctx = ctx_from_connector(connector);
+
+        /*
+         * connection request would come from user side
+         * to do hotplug through specific ioctl.
+         */
+        return ctx->connected ? connector_status_connected :
+                        connector_status_disconnected;
+}
+
+static void vidi_connector_destroy(struct drm_connector *connector)
+{
+}
+
+static struct drm_connector_funcs vidi_connector_funcs = {
+        .dpms = drm_helper_connector_dpms,
+        .fill_modes = drm_helper_probe_single_connector_modes,
+        .detect = vidi_detect,
+        .destroy = vidi_connector_destroy,
+};
+
+static int vidi_get_modes(struct drm_connector *connector)
+{
+        struct vidi_context *ctx = ctx_from_connector(connector);
+        struct edid *edid;
+        int edid_len;
+
+        /*
+         * the edid data comes from user side and it would be set
+         * to ctx->raw_edid through specific ioctl.
+         */
+        if (!ctx->raw_edid) {
+                DRM_DEBUG_KMS("raw_edid is null.\n");
+                return -EFAULT;
+        }
+
+        edid_len = (1 + ctx->raw_edid->extensions) * EDID_LENGTH;
+        edid = kmemdup(ctx->raw_edid, edid_len, GFP_KERNEL);
+        if (!edid) {
+                DRM_DEBUG_KMS("failed to allocate edid\n");
+                return -ENOMEM;
+        }
+
+        drm_mode_connector_update_edid_property(connector, edid);
+
+        return drm_add_edid_modes(connector, edid);
+}
+
+static int vidi_mode_valid(struct drm_connector *connector,
+                        struct drm_display_mode *mode)
+{
+        return MODE_OK;
+}
+
+static struct drm_encoder *vidi_best_encoder(struct drm_connector *connector)
+{
+        struct vidi_context *ctx = ctx_from_connector(connector);
+
+        return ctx->encoder;
+}
+
+static struct drm_connector_helper_funcs vidi_connector_helper_funcs = {
+        .get_modes = vidi_get_modes,
+        .mode_valid = vidi_mode_valid,
+        .best_encoder = vidi_best_encoder,
+};
+
+static int vidi_create_connector(struct exynos_drm_display *display,
+                                struct drm_encoder *encoder)
+{
+        struct vidi_context *ctx = display->ctx;
+        struct drm_connector *connector = &ctx->connector;
+        int ret;
+
+        ctx->encoder = encoder;
+        connector->polled = DRM_CONNECTOR_POLL_HPD;
+
+        ret = drm_connector_init(ctx->drm_dev, connector,
+                        &vidi_connector_funcs, DRM_MODE_CONNECTOR_VIRTUAL);
+        if (ret) {
+                DRM_ERROR("Failed to initialize connector with drm\n");
+                return ret;
+        }
+
+        drm_connector_helper_add(connector, &vidi_connector_helper_funcs);
+        drm_sysfs_connector_add(connector);
+        drm_mode_connector_attach_encoder(connector, encoder);
 
         return 0;
 }
 
+
+static struct exynos_drm_display_ops vidi_display_ops = {
+        .create_connector = vidi_create_connector,
+};
+
+static struct exynos_drm_display vidi_display = {
+        .type = EXYNOS_DISPLAY_TYPE_VIDI,
+        .ops = &vidi_display_ops,
+};
+
 static int vidi_probe(struct platform_device *pdev)
 {
         struct device *dev = &pdev->dev;
         struct vidi_context *ctx;
-        struct exynos_drm_subdrv *subdrv;
         int ret;
 
         ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
@@ -559,21 +600,19 @@ static int vidi_probe(struct platform_device *pdev)
 
         INIT_WORK(&ctx->work, vidi_fake_vblank_handler);
 
-        subdrv = &ctx->subdrv;
-        subdrv->dev = dev;
-        subdrv->manager = &vidi_manager;
-        subdrv->probe = vidi_subdrv_probe;
-        subdrv->remove = vidi_subdrv_remove;
+        vidi_manager.ctx = ctx;
+        vidi_display.ctx = ctx;
 
         mutex_init(&ctx->lock);
 
-        platform_set_drvdata(pdev, ctx);
+        platform_set_drvdata(pdev, &vidi_manager);
 
         ret = device_create_file(dev, &dev_attr_connection);
         if (ret < 0)
                 DRM_INFO("failed to create connection sysfs.\n");
 
-        exynos_drm_subdrv_register(subdrv);
+        exynos_drm_manager_register(&vidi_manager);
+        exynos_drm_display_register(&vidi_display);
 
         return 0;
 }
@@ -582,7 +621,8 @@ static int vidi_remove(struct platform_device *pdev)
 {
         struct vidi_context *ctx = platform_get_drvdata(pdev);
 
-        exynos_drm_subdrv_unregister(&ctx->subdrv);
+        exynos_drm_display_unregister(&vidi_display);
+        exynos_drm_manager_unregister(&vidi_manager);
 
         if (ctx->raw_edid != (struct edid *)fake_edid_info) {
                 kfree(ctx->raw_edid);
@@ -592,32 +632,11 @@ static int vidi_remove(struct platform_device *pdev)
         return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int vidi_suspend(struct device *dev)
-{
-        struct vidi_context *ctx = get_vidi_context(dev);
-
-        return vidi_power_on(ctx, false);
-}
-
-static int vidi_resume(struct device *dev)
-{
-        struct vidi_context *ctx = get_vidi_context(dev);
-
-        return vidi_power_on(ctx, true);
-}
-#endif
-
-static const struct dev_pm_ops vidi_pm_ops = {
-        SET_SYSTEM_SLEEP_PM_OPS(vidi_suspend, vidi_resume)
-};
-
 struct platform_driver vidi_driver = {
         .probe          = vidi_probe,
         .remove         = vidi_remove,
         .driver         = {
                 .name   = "exynos-drm-vidi",
                 .owner  = THIS_MODULE,
-                .pm     = &vidi_pm_ops,
         },
 };
diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c
index c021ddc1ffb4..9a6d652a3ef2 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmi.c
@@ -33,38 +33,42 @@
 #include <linux/regulator/consumer.h>
 #include <linux/io.h>
 #include <linux/of.h>
+#include <linux/i2c.h>
 #include <linux/of_gpio.h>
 #include <linux/hdmi.h>
 
 #include <drm/exynos_drm.h>
 
 #include "exynos_drm_drv.h"
-#include "exynos_drm_hdmi.h"
-
-#include "exynos_hdmi.h"
+#include "exynos_mixer.h"
 
 #include <linux/gpio.h>
 #include <media/s5p_hdmi.h>
 
-#define MAX_WIDTH               1920
-#define MAX_HEIGHT              1080
-#define get_hdmi_context(dev)   platform_get_drvdata(to_platform_device(dev))
+#define get_hdmi_display(dev)   platform_get_drvdata(to_platform_device(dev))
+#define ctx_from_connector(c)   container_of(c, struct hdmi_context, connector)
 
 /* AVI header and aspect ratio */
 #define HDMI_AVI_VERSION                0x02
 #define HDMI_AVI_LENGTH         0x0D
-#define AVI_PIC_ASPECT_RATIO_16_9       (2 << 4)
-#define AVI_SAME_AS_PIC_ASPECT_RATIO    8
 
 /* AUI header info */
 #define HDMI_AUI_VERSION        0x01
 #define HDMI_AUI_LENGTH 0x0A
+#define AVI_SAME_AS_PIC_ASPECT_RATIO 0x8
+#define AVI_4_3_CENTER_RATIO    0x9
+#define AVI_16_9_CENTER_RATIO   0xa
 
 enum hdmi_type {
         HDMI_TYPE13,
         HDMI_TYPE14,
 };
 
+struct hdmi_driver_data {
+        unsigned int type;
+        unsigned int is_apb_phy:1;
+};
+
 struct hdmi_resources {
         struct clk                      *hdmi;
         struct clk                      *sclk_hdmi;
@@ -162,6 +166,7 @@ struct hdmi_v14_conf {
 struct hdmi_conf_regs {
         int pixel_clock;
         int cea_video_id;
+        enum hdmi_picture_aspect aspect_ratio;
         union {
                 struct hdmi_v13_conf v13_conf;
                 struct hdmi_v14_conf v14_conf;
@@ -171,16 +176,17 @@ struct hdmi_conf_regs {
 struct hdmi_context {
         struct device                   *dev;
         struct drm_device               *drm_dev;
+        struct drm_connector            connector;
+        struct drm_encoder              *encoder;
         bool                            hpd;
         bool                            powered;
         bool                            dvi_mode;
         struct mutex                    hdmi_mutex;
 
         void __iomem                    *regs;
-        void                            *parent_ctx;
         int                             irq;
 
-        struct i2c_client               *ddc_port;
+        struct i2c_adapter              *ddc_adpt;
         struct i2c_client               *hdmiphy_port;
 
         /* current hdmiphy conf regs */
@@ -198,6 +204,14 @@ struct hdmiphy_config {
         u8 conf[32];
 };
 
+struct hdmi_driver_data exynos4212_hdmi_driver_data = {
+        .type   = HDMI_TYPE14,
+};
+
+struct hdmi_driver_data exynos5_hdmi_driver_data = {
+        .type   = HDMI_TYPE14,
+};
+
 /* list of phy config settings */
 static const struct hdmiphy_config hdmiphy_v13_configs[] = {
         {
@@ -303,6 +317,24 @@ static const struct hdmiphy_config hdmiphy_v14_configs[] = {
                 },
         },
         {
+                .pixel_clock = 71000000,
+                .conf = {
+                        0x01, 0x91, 0x1e, 0x15, 0x40, 0x3c, 0xce, 0x08,
+                        0x04, 0x20, 0xb2, 0xd8, 0x45, 0xa0, 0xac, 0x80,
+                        0x06, 0x80, 0x11, 0x04, 0x02, 0x22, 0x44, 0x86,
+                        0x54, 0xad, 0x24, 0x01, 0x00, 0x00, 0x01, 0x80,
+                },
+        },
+        {
+                .pixel_clock = 73250000,
+                .conf = {
+                        0x01, 0xd1, 0x1f, 0x15, 0x40, 0x18, 0xe9, 0x08,
+                        0x02, 0xa0, 0xb7, 0xd8, 0x45, 0xa0, 0xac, 0x80,
+                        0x06, 0x80, 0x11, 0x04, 0x02, 0x22, 0x44, 0x86,
+                        0x54, 0xa8, 0x24, 0x01, 0x00, 0x00, 0x01, 0x80,
+                },
+        },
+        {
                 .pixel_clock = 74176000,
                 .conf = {
                         0x01, 0xd1, 0x3e, 0x35, 0x40, 0x5b, 0xde, 0x08,
@@ -330,6 +362,15 @@ static const struct hdmiphy_config hdmiphy_v14_configs[] = {
                 },
         },
         {
+                .pixel_clock = 88750000,
+                .conf = {
+                        0x01, 0x91, 0x25, 0x17, 0x40, 0x30, 0xfe, 0x08,
+                        0x06, 0x20, 0xde, 0xd8, 0x45, 0xa0, 0xac, 0x80,
+                        0x06, 0x80, 0x11, 0x04, 0x02, 0x22, 0x44, 0x86,
+                        0x54, 0x8a, 0x24, 0x01, 0x00, 0x00, 0x01, 0x80,
+                },
+        },
+        {
                 .pixel_clock = 106500000,
                 .conf = {
                         0x01, 0xd1, 0x2c, 0x12, 0x40, 0x0c, 0x09, 0x08,
@@ -348,6 +389,24 @@ static const struct hdmiphy_config hdmiphy_v14_configs[] = {
                 },
         },
         {
+                .pixel_clock = 115500000,
+                .conf = {
+                        0x01, 0xd1, 0x30, 0x1a, 0x40, 0x40, 0x10, 0x04,
+                        0x04, 0xa0, 0x21, 0xd9, 0x45, 0xa0, 0xac, 0x80,
+                        0x06, 0x80, 0x11, 0x04, 0x02, 0x22, 0x44, 0x86,
+                        0x54, 0xaa, 0x25, 0x03, 0x00, 0x00, 0x01, 0x80,
+                },
+        },
+        {
+                .pixel_clock = 119000000,
+                .conf = {
+                        0x01, 0x91, 0x32, 0x14, 0x40, 0x60, 0xd8, 0x08,
+                        0x06, 0x20, 0x2a, 0xd9, 0x45, 0xa0, 0xac, 0x80,
+                        0x06, 0x80, 0x11, 0x04, 0x02, 0x22, 0x44, 0x86,
+                        0x54, 0x9d, 0x25, 0x03, 0x00, 0x00, 0x01, 0x80,
+                },
+        },
+        {
                 .pixel_clock = 146250000,
                 .conf = {
                         0x01, 0xd1, 0x3d, 0x15, 0x40, 0x18, 0xfd, 0x08,
@@ -668,7 +727,6 @@ static void hdmi_reg_infoframe(struct hdmi_context *hdata,
 {
         u32 hdr_sum;
         u8 chksum;
-        u32 aspect_ratio;
         u32 mod;
         u32 vic;
 
@@ -697,10 +755,28 @@ static void hdmi_reg_infoframe(struct hdmi_context *hdata,
                         AVI_ACTIVE_FORMAT_VALID |
                         AVI_UNDERSCANNED_DISPLAY_VALID);
 
-                aspect_ratio = AVI_PIC_ASPECT_RATIO_16_9;
-
-                hdmi_reg_writeb(hdata, HDMI_AVI_BYTE(2), aspect_ratio |
-                                AVI_SAME_AS_PIC_ASPECT_RATIO);
+                /*
+                 * Set the aspect ratio as per the mode, mentioned in
+                 * Table 9 AVI InfoFrame Data Byte 2 of CEA-861-D Standard
+                 */
+                switch (hdata->mode_conf.aspect_ratio) {
+                case HDMI_PICTURE_ASPECT_4_3:
+                        hdmi_reg_writeb(hdata, HDMI_AVI_BYTE(2),
+                                        hdata->mode_conf.aspect_ratio |
+                                        AVI_4_3_CENTER_RATIO);
+                        break;
+                case HDMI_PICTURE_ASPECT_16_9:
+                        hdmi_reg_writeb(hdata, HDMI_AVI_BYTE(2),
+                                        hdata->mode_conf.aspect_ratio |
+                                        AVI_16_9_CENTER_RATIO);
+                        break;
+                case HDMI_PICTURE_ASPECT_NONE:
+                default:
+                        hdmi_reg_writeb(hdata, HDMI_AVI_BYTE(2),
+                                        hdata->mode_conf.aspect_ratio |
+                                        AVI_SAME_AS_PIC_ASPECT_RATIO);
+                        break;
+                }
 
                 vic = hdata->mode_conf.cea_video_id;
                 hdmi_reg_writeb(hdata, HDMI_AVI_BYTE(4), vic);
@@ -728,31 +804,46 @@ static void hdmi_reg_infoframe(struct hdmi_context *hdata,
         }
 }
 
-static bool hdmi_is_connected(void *ctx)
+static enum drm_connector_status hdmi_detect(struct drm_connector *connector,
+                                bool force)
 {
-        struct hdmi_context *hdata = ctx;
+        struct hdmi_context *hdata = ctx_from_connector(connector);
+
+        return hdata->hpd ? connector_status_connected :
+                        connector_status_disconnected;
+}
 
-        return hdata->hpd;
+static void hdmi_connector_destroy(struct drm_connector *connector)
+{
 }
 
-static struct edid *hdmi_get_edid(void *ctx, struct drm_connector *connector)
+static struct drm_connector_funcs hdmi_connector_funcs = {
+        .dpms = drm_helper_connector_dpms,
+        .fill_modes = drm_helper_probe_single_connector_modes,
+        .detect = hdmi_detect,
+        .destroy = hdmi_connector_destroy,
+};
+
+static int hdmi_get_modes(struct drm_connector *connector)
 {
-        struct edid *raw_edid;
-        struct hdmi_context *hdata = ctx;
+        struct hdmi_context *hdata = ctx_from_connector(connector);
+        struct edid *edid;
 
-        if (!hdata->ddc_port)
-                return ERR_PTR(-ENODEV);
+        if (!hdata->ddc_adpt)
+                return -ENODEV;
 
-        raw_edid = drm_get_edid(connector, hdata->ddc_port->adapter);
-        if (!raw_edid)
-                return ERR_PTR(-ENODEV);
+        edid = drm_get_edid(connector, hdata->ddc_adpt);
+        if (!edid)
+                return -ENODEV;
 
-        hdata->dvi_mode = !drm_detect_hdmi_monitor(raw_edid);
+        hdata->dvi_mode = !drm_detect_hdmi_monitor(edid);
         DRM_DEBUG_KMS("%s : width[%d] x height[%d]\n",
                 (hdata->dvi_mode ? "dvi monitor" : "hdmi monitor"),
-                raw_edid->width_cm, raw_edid->height_cm);
+                edid->width_cm, edid->height_cm);
 
-        return raw_edid;
+        drm_mode_connector_update_edid_property(connector, edid);
+
+        return drm_add_edid_modes(connector, edid);
 }
 
 static int hdmi_find_phy_conf(struct hdmi_context *hdata, u32 pixel_clock)
@@ -777,9 +868,10 @@ static int hdmi_find_phy_conf(struct hdmi_context *hdata, u32 pixel_clock)
         return -EINVAL;
 }
 
-static int hdmi_check_mode(void *ctx, struct drm_display_mode *mode)
+static int hdmi_mode_valid(struct drm_connector *connector,
+                        struct drm_display_mode *mode)
 {
-        struct hdmi_context *hdata = ctx;
+        struct hdmi_context *hdata = ctx_from_connector(connector);
         int ret;
 
         DRM_DEBUG_KMS("xres=%d, yres=%d, refresh=%d, intl=%d clock=%d\n",
@@ -787,12 +879,103 @@ static int hdmi_check_mode(void *ctx, struct drm_display_mode *mode)
                 (mode->flags & DRM_MODE_FLAG_INTERLACE) ? true :
                 false, mode->clock * 1000);
 
+        ret = mixer_check_mode(mode);
+        if (ret)
+                return MODE_BAD;
+
         ret = hdmi_find_phy_conf(hdata, mode->clock * 1000);
         if (ret < 0)
+                return MODE_BAD;
+
+        return MODE_OK;
+}
+
+static struct drm_encoder *hdmi_best_encoder(struct drm_connector *connector)
+{
+        struct hdmi_context *hdata = ctx_from_connector(connector);
+
+        return hdata->encoder;
+}
+
+static struct drm_connector_helper_funcs hdmi_connector_helper_funcs = {
+        .get_modes = hdmi_get_modes,
+        .mode_valid = hdmi_mode_valid,
+        .best_encoder = hdmi_best_encoder,
+};
+
+static int hdmi_create_connector(struct exynos_drm_display *display,
+                        struct drm_encoder *encoder)
+{
+        struct hdmi_context *hdata = display->ctx;
+        struct drm_connector *connector = &hdata->connector;
+        int ret;
+
+        hdata->encoder = encoder;
+        connector->interlace_allowed = true;
+        connector->polled = DRM_CONNECTOR_POLL_HPD;
+
+        ret = drm_connector_init(hdata->drm_dev, connector,
+                        &hdmi_connector_funcs, DRM_MODE_CONNECTOR_HDMIA);
+        if (ret) {
+                DRM_ERROR("Failed to initialize connector with drm\n");
                 return ret;
+        }
+
+        drm_connector_helper_add(connector, &hdmi_connector_helper_funcs);
+        drm_sysfs_connector_add(connector);
+        drm_mode_connector_attach_encoder(connector, encoder);
+
+        return 0;
+}
+
+static int hdmi_initialize(struct exynos_drm_display *display,
+                        struct drm_device *drm_dev)
+{
+        struct hdmi_context *hdata = display->ctx;
+
+        hdata->drm_dev = drm_dev;
+
         return 0;
 }
 
+static void hdmi_mode_fixup(struct exynos_drm_display *display,
+                                struct drm_connector *connector,
+                                const struct drm_display_mode *mode,
+                                struct drm_display_mode *adjusted_mode)
+{
+        struct drm_display_mode *m;
+        int mode_ok;
+
+        DRM_DEBUG_KMS("%s\n", __FILE__);
+
+        drm_mode_set_crtcinfo(adjusted_mode, 0);
+
+        mode_ok = hdmi_mode_valid(connector, adjusted_mode);
+
+        /* just return if user desired mode exists. */
+        if (mode_ok == MODE_OK)
+                return;
+
+        /*
+         * otherwise, find the most suitable mode among modes and change it
+         * to adjusted_mode.
+         */
+        list_for_each_entry(m, &connector->modes, head) {
+                mode_ok = hdmi_mode_valid(connector, m);
+
+                if (mode_ok == MODE_OK) {
+                        DRM_INFO("desired mode doesn't exist so\n");
+                        DRM_INFO("use the most suitable mode among modes.\n");
+
+                        DRM_DEBUG_KMS("Adjusted Mode: [%d]x[%d] [%d]Hz\n",
+                                m->hdisplay, m->vdisplay, m->vrefresh);
+
+                        drm_mode_copy(adjusted_mode, m);
+                        break;
+                }
+        }
+}
+
 static void hdmi_set_acr(u32 freq, u8 *acr)
 {
         u32 n, cts;
@@ -1421,6 +1604,7 @@ static void hdmi_v13_mode_set(struct hdmi_context *hdata,
         hdata->mode_conf.cea_video_id =
                 drm_match_cea_mode((struct drm_display_mode *)m);
         hdata->mode_conf.pixel_clock = m->clock * 1000;
+        hdata->mode_conf.aspect_ratio = m->picture_aspect_ratio;
 
         hdmi_set_reg(core->h_blank, 2, m->htotal - m->hdisplay);
         hdmi_set_reg(core->h_v_line, 3, (m->htotal << 12) | m->vtotal);
@@ -1517,6 +1701,7 @@ static void hdmi_v14_mode_set(struct hdmi_context *hdata,
         hdata->mode_conf.cea_video_id =
                 drm_match_cea_mode((struct drm_display_mode *)m);
         hdata->mode_conf.pixel_clock = m->clock * 1000;
+        hdata->mode_conf.aspect_ratio = m->picture_aspect_ratio;
 
         hdmi_set_reg(core->h_blank, 2, m->htotal - m->hdisplay);
         hdmi_set_reg(core->v_line, 2, m->vtotal);
@@ -1618,9 +1803,10 @@ static void hdmi_v14_mode_set(struct hdmi_context *hdata,
         hdmi_set_reg(tg->tg_3d, 1, 0x0);
 }
 
-static void hdmi_mode_set(void *ctx, struct drm_display_mode *mode)
+static void hdmi_mode_set(struct exynos_drm_display *display,
+                        struct drm_display_mode *mode)
 {
-        struct hdmi_context *hdata = ctx;
+        struct hdmi_context *hdata = display->ctx;
         struct drm_display_mode *m = mode;
 
         DRM_DEBUG_KMS("xres=%d, yres=%d, refresh=%d, intl=%s\n",
@@ -1634,16 +1820,9 @@ static void hdmi_mode_set(void *ctx, struct drm_display_mode *mode)
                 hdmi_v14_mode_set(hdata, mode);
 }
 
-static void hdmi_get_max_resol(void *ctx, unsigned int *width,
-                                        unsigned int *height)
-{
-        *width = MAX_WIDTH;
-        *height = MAX_HEIGHT;
-}
-
-static void hdmi_commit(void *ctx)
+static void hdmi_commit(struct exynos_drm_display *display)
 {
-        struct hdmi_context *hdata = ctx;
+        struct hdmi_context *hdata = display->ctx;
 
         mutex_lock(&hdata->hdmi_mutex);
         if (!hdata->powered) {
@@ -1655,8 +1834,9 @@ static void hdmi_commit(void *ctx)
         hdmi_conf_apply(hdata);
 }
 
-static void hdmi_poweron(struct hdmi_context *hdata)
+static void hdmi_poweron(struct exynos_drm_display *display)
 {
+        struct hdmi_context *hdata = display->ctx;
         struct hdmi_resources *res = &hdata->res;
 
         mutex_lock(&hdata->hdmi_mutex);
@@ -1669,6 +1849,8 @@ static void hdmi_poweron(struct hdmi_context *hdata)
 
         mutex_unlock(&hdata->hdmi_mutex);
 
+        pm_runtime_get_sync(hdata->dev);
+
         if (regulator_bulk_enable(res->regul_count, res->regul_bulk))
                 DRM_DEBUG_KMS("failed to enable regulator bulk\n");
 
@@ -1677,10 +1859,12 @@ static void hdmi_poweron(struct hdmi_context *hdata)
         clk_prepare_enable(res->sclk_hdmi);
 
         hdmiphy_poweron(hdata);
+        hdmi_commit(display);
 }
 
-static void hdmi_poweroff(struct hdmi_context *hdata)
+static void hdmi_poweroff(struct exynos_drm_display *display)
 {
+        struct hdmi_context *hdata = display->ctx;
         struct hdmi_resources *res = &hdata->res;
 
         mutex_lock(&hdata->hdmi_mutex);
@@ -1700,30 +1884,27 @@ static void hdmi_poweroff(struct hdmi_context *hdata)
         clk_disable_unprepare(res->hdmiphy);
         regulator_bulk_disable(res->regul_count, res->regul_bulk);
 
-        mutex_lock(&hdata->hdmi_mutex);
+        pm_runtime_put_sync(hdata->dev);
 
+        mutex_lock(&hdata->hdmi_mutex);
         hdata->powered = false;
 
 out:
         mutex_unlock(&hdata->hdmi_mutex);
 }
 
-static void hdmi_dpms(void *ctx, int mode)
+static void hdmi_dpms(struct exynos_drm_display *display, int mode)
 {
-        struct hdmi_context *hdata = ctx;
-
         DRM_DEBUG_KMS("mode %d\n", mode);
 
         switch (mode) {
         case DRM_MODE_DPMS_ON:
-                if (pm_runtime_suspended(hdata->dev))
-                        pm_runtime_get_sync(hdata->dev);
+                hdmi_poweron(display);
                 break;
         case DRM_MODE_DPMS_STANDBY:
         case DRM_MODE_DPMS_SUSPEND:
         case DRM_MODE_DPMS_OFF:
-                if (!pm_runtime_suspended(hdata->dev))
-                        pm_runtime_put_sync(hdata->dev);
+                hdmi_poweroff(display);
                 break;
         default:
                 DRM_DEBUG_KMS("unknown dpms mode: %d\n", mode);
@@ -1731,30 +1912,30 @@ static void hdmi_dpms(void *ctx, int mode)
         }
 }
 
-static struct exynos_hdmi_ops hdmi_ops = {
-        /* display */
-        .is_connected   = hdmi_is_connected,
-        .get_edid       = hdmi_get_edid,
-        .check_mode     = hdmi_check_mode,
-
-        /* manager */
+static struct exynos_drm_display_ops hdmi_display_ops = {
+        .initialize     = hdmi_initialize,
+        .create_connector = hdmi_create_connector,
+        .mode_fixup     = hdmi_mode_fixup,
         .mode_set       = hdmi_mode_set,
-        .get_max_resol  = hdmi_get_max_resol,
-        .commit         = hdmi_commit,
         .dpms           = hdmi_dpms,
+        .commit         = hdmi_commit,
+};
+
+static struct exynos_drm_display hdmi_display = {
+        .type = EXYNOS_DISPLAY_TYPE_HDMI,
+        .ops = &hdmi_display_ops,
 };
 
 static irqreturn_t hdmi_irq_thread(int irq, void *arg)
 {
-        struct exynos_drm_hdmi_context *ctx = arg;
-        struct hdmi_context *hdata = ctx->ctx;
+        struct hdmi_context *hdata = arg;
 
         mutex_lock(&hdata->hdmi_mutex);
         hdata->hpd = gpio_get_value(hdata->hpd_gpio);
         mutex_unlock(&hdata->hdmi_mutex);
 
-        if (ctx->drm_dev)
-                drm_helper_hpd_irq_event(ctx->drm_dev);
+        if (hdata->drm_dev)
+                drm_helper_hpd_irq_event(hdata->drm_dev);
 
         return IRQ_HANDLED;
 }
@@ -1830,20 +2011,6 @@ fail:
         return -ENODEV;
 }
 
-static struct i2c_client *hdmi_ddc, *hdmi_hdmiphy;
-
-void hdmi_attach_ddc_client(struct i2c_client *ddc)
-{
-        if (ddc)
-                hdmi_ddc = ddc;
-}
-
-void hdmi_attach_hdmiphy_client(struct i2c_client *hdmiphy)
-{
-        if (hdmiphy)
-                hdmi_hdmiphy = hdmiphy;
-}
-
 static struct s5p_hdmi_platform_data *drm_hdmi_dt_parse_pdata
                                         (struct device *dev)
 {
@@ -1871,10 +2038,10 @@ err_data:
 static struct of_device_id hdmi_match_types[] = {
         {
                 .compatible = "samsung,exynos5-hdmi",
-                .data   = (void *)HDMI_TYPE14,
+                .data = &exynos5_hdmi_driver_data,
         }, {
                 .compatible = "samsung,exynos4212-hdmi",
-                .data   = (void *)HDMI_TYPE14,
+                .data = &exynos4212_hdmi_driver_data,
         }, {
                 /* end node */
         }
@@ -1883,11 +2050,12 @@ static struct of_device_id hdmi_match_types[] = {
 static int hdmi_probe(struct platform_device *pdev)
 {
         struct device *dev = &pdev->dev;
-        struct exynos_drm_hdmi_context *drm_hdmi_ctx;
         struct hdmi_context *hdata;
         struct s5p_hdmi_platform_data *pdata;
         struct resource *res;
         const struct of_device_id *match;
+        struct device_node *ddc_node, *phy_node;
+        struct hdmi_driver_data *drv_data;
         int ret;
 
          if (!dev->of_node)
@@ -1897,25 +2065,20 @@ static int hdmi_probe(struct platform_device *pdev)
         if (!pdata)
                 return -EINVAL;
 
-        drm_hdmi_ctx = devm_kzalloc(dev, sizeof(*drm_hdmi_ctx), GFP_KERNEL);
-        if (!drm_hdmi_ctx)
-                return -ENOMEM;
-
         hdata = devm_kzalloc(dev, sizeof(struct hdmi_context), GFP_KERNEL);
         if (!hdata)
                 return -ENOMEM;
 
         mutex_init(&hdata->hdmi_mutex);
 
-        drm_hdmi_ctx->ctx = (void *)hdata;
-        hdata->parent_ctx = (void *)drm_hdmi_ctx;
-
-        platform_set_drvdata(pdev, drm_hdmi_ctx);
+        platform_set_drvdata(pdev, &hdmi_display);
 
         match = of_match_node(hdmi_match_types, dev->of_node);
         if (!match)
                 return -ENODEV;
-        hdata->type = (enum hdmi_type)match->data;
+
+        drv_data = (struct hdmi_driver_data *)match->data;
+        hdata->type = drv_data->type;
 
         hdata->hpd_gpio = pdata->hpd_gpio;
         hdata->dev = dev;
@@ -1938,21 +2101,34 @@ static int hdmi_probe(struct platform_device *pdev)
         }
 
         /* DDC i2c driver */
-        if (i2c_add_driver(&ddc_driver)) {
-                DRM_ERROR("failed to register ddc i2c driver\n");
-                return -ENOENT;
+        ddc_node = of_parse_phandle(dev->of_node, "ddc", 0);
+        if (!ddc_node) {
+                DRM_ERROR("Failed to find ddc node in device tree\n");
+                return -ENODEV;
+        }
+        hdata->ddc_adpt = of_find_i2c_adapter_by_node(ddc_node);
+        if (!hdata->ddc_adpt) {
+                DRM_ERROR("Failed to get ddc i2c adapter by node\n");
+                return -ENODEV;
         }
 
-        hdata->ddc_port = hdmi_ddc;
+        /* Not support APB PHY yet. */
+        if (drv_data->is_apb_phy)
+                return -EPERM;
 
         /* hdmiphy i2c driver */
-        if (i2c_add_driver(&hdmiphy_driver)) {
-                DRM_ERROR("failed to register hdmiphy i2c driver\n");
-                ret = -ENOENT;
+        phy_node = of_parse_phandle(dev->of_node, "phy", 0);
+        if (!phy_node) {
+                DRM_ERROR("Failed to find hdmiphy node in device tree\n");
+                ret = -ENODEV;
+                goto err_ddc;
+        }
+        hdata->hdmiphy_port = of_find_i2c_device_by_node(phy_node);
+        if (!hdata->hdmiphy_port) {
+                DRM_ERROR("Failed to get hdmi phy i2c client from node\n");
+                ret = -ENODEV;
                 goto err_ddc;
         }
-
-        hdata->hdmiphy_port = hdmi_hdmiphy;
 
         hdata->irq = gpio_to_irq(hdata->hpd_gpio);
         if (hdata->irq < 0) {
@@ -1966,119 +2142,45 @@ static int hdmi_probe(struct platform_device *pdev)
         ret = devm_request_threaded_irq(dev, hdata->irq, NULL,
                         hdmi_irq_thread, IRQF_TRIGGER_RISING |
                         IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
-                        "hdmi", drm_hdmi_ctx);
+                        "hdmi", hdata);
         if (ret) {
                 DRM_ERROR("failed to register hdmi interrupt\n");
                 goto err_hdmiphy;
         }
 
-        /* Attach HDMI Driver to common hdmi. */
-        exynos_hdmi_drv_attach(drm_hdmi_ctx);
-
-        /* register specific callbacks to common hdmi. */
-        exynos_hdmi_ops_register(&hdmi_ops);
-
         pm_runtime_enable(dev);
 
+        hdmi_display.ctx = hdata;
+        exynos_drm_display_register(&hdmi_display);
+
         return 0;
 
 err_hdmiphy:
-        i2c_del_driver(&hdmiphy_driver);
+        put_device(&hdata->hdmiphy_port->dev);
 err_ddc:
-        i2c_del_driver(&ddc_driver);
+        put_device(&hdata->ddc_adpt->dev);
         return ret;
 }
 
 static int hdmi_remove(struct platform_device *pdev)
 {
         struct device *dev = &pdev->dev;
+        struct exynos_drm_display *display = get_hdmi_display(dev);
+        struct hdmi_context *hdata = display->ctx;
 
-        pm_runtime_disable(dev);
-
-        /* hdmiphy i2c driver */
-        i2c_del_driver(&hdmiphy_driver);
-        /* DDC i2c driver */
-        i2c_del_driver(&ddc_driver);
-
-        return 0;
-}
-
-#ifdef CONFIG_PM_SLEEP
-static int hdmi_suspend(struct device *dev)
-{
-        struct exynos_drm_hdmi_context *ctx = get_hdmi_context(dev);
-        struct hdmi_context *hdata = ctx->ctx;
-
-        disable_irq(hdata->irq);
-
-        hdata->hpd = false;
-        if (ctx->drm_dev)
-                drm_helper_hpd_irq_event(ctx->drm_dev);
-
-        if (pm_runtime_suspended(dev)) {
-                DRM_DEBUG_KMS("Already suspended\n");
-                return 0;
-        }
-
-        hdmi_poweroff(hdata);
+        put_device(&hdata->hdmiphy_port->dev);
+        put_device(&hdata->ddc_adpt->dev);
+        pm_runtime_disable(&pdev->dev);
 
         return 0;
 }
 
-static int hdmi_resume(struct device *dev)
-{
-        struct exynos_drm_hdmi_context *ctx = get_hdmi_context(dev);
-        struct hdmi_context *hdata = ctx->ctx;
-
-        hdata->hpd = gpio_get_value(hdata->hpd_gpio);
-
-        enable_irq(hdata->irq);
-
-        if (!pm_runtime_suspended(dev)) {
-                DRM_DEBUG_KMS("Already resumed\n");
-                return 0;
-        }
-
-        hdmi_poweron(hdata);
-
-        return 0;
-}
-#endif
-
-#ifdef CONFIG_PM_RUNTIME
-static int hdmi_runtime_suspend(struct device *dev)
-{
-        struct exynos_drm_hdmi_context *ctx = get_hdmi_context(dev);
-        struct hdmi_context *hdata = ctx->ctx;
-
-        hdmi_poweroff(hdata);
-
-        return 0;
-}
-
-static int hdmi_runtime_resume(struct device *dev)
-{
-        struct exynos_drm_hdmi_context *ctx = get_hdmi_context(dev);
-        struct hdmi_context *hdata = ctx->ctx;
-
-        hdmi_poweron(hdata);
-
-        return 0;
-}
-#endif
-
-static const struct dev_pm_ops hdmi_pm_ops = {
-        SET_SYSTEM_SLEEP_PM_OPS(hdmi_suspend, hdmi_resume)
-        SET_RUNTIME_PM_OPS(hdmi_runtime_suspend, hdmi_runtime_resume, NULL)
-};
-
 struct platform_driver hdmi_driver = {
         .probe          = hdmi_probe,
         .remove         = hdmi_remove,
         .driver         = {
                 .name   = "exynos-hdmi",
                 .owner  = THIS_MODULE,
-                .pm     = &hdmi_pm_ops,
                 .of_match_table = hdmi_match_types,
         },
 };
diff --git a/drivers/gpu/drm/exynos/exynos_mixer.c b/drivers/gpu/drm/exynos/exynos_mixer.c
index 2dfa48c76f54..ce288818d2c0 100644
--- a/drivers/gpu/drm/exynos/exynos_mixer.c
+++ b/drivers/gpu/drm/exynos/exynos_mixer.c
@@ -36,10 +36,13 @@
 
 #include "exynos_drm_drv.h"
 #include "exynos_drm_crtc.h"
-#include "exynos_drm_hdmi.h"
 #include "exynos_drm_iommu.h"
+#include "exynos_mixer.h"
 
-#define get_mixer_context(dev)  platform_get_drvdata(to_platform_device(dev))
+#define get_mixer_manager(dev)  platform_get_drvdata(to_platform_device(dev))
+
+#define MIXER_WIN_NR            3
+#define MIXER_DEFAULT_WIN       0
 
 struct hdmi_win_data {
         dma_addr_t              dma_addr;
@@ -82,6 +85,7 @@ enum mixer_version_id {
 };
 
 struct mixer_context {
+        struct platform_device *pdev;
         struct device           *dev;
         struct drm_device       *drm_dev;
         int                     pipe;
@@ -94,7 +98,6 @@ struct mixer_context {
         struct mixer_resources  mixer_res;
         struct hdmi_win_data    win_data[MIXER_WIN_NR];
         enum mixer_version_id   mxr_ver;
-        void                    *parent_ctx;
         wait_queue_head_t       wait_vsync_queue;
         atomic_t                wait_vsync_event;
 };
@@ -685,31 +688,196 @@ static void mixer_win_reset(struct mixer_context *ctx)
         spin_unlock_irqrestore(&res->reg_slock, flags);
 }
 
-static int mixer_iommu_on(void *ctx, bool enable)
+static irqreturn_t mixer_irq_handler(int irq, void *arg)
+{
+        struct mixer_context *ctx = arg;
+        struct mixer_resources *res = &ctx->mixer_res;
+        u32 val, base, shadow;
+
+        spin_lock(&res->reg_slock);
+
+        /* read interrupt status for handling and clearing flags for VSYNC */
+        val = mixer_reg_read(res, MXR_INT_STATUS);
+
+        /* handling VSYNC */
+        if (val & MXR_INT_STATUS_VSYNC) {
+                /* interlace scan need to check shadow register */
+                if (ctx->interlace) {
+                        base = mixer_reg_read(res, MXR_GRAPHIC_BASE(0));
+                        shadow = mixer_reg_read(res, MXR_GRAPHIC_BASE_S(0));
+                        if (base != shadow)
+                                goto out;
+
+                        base = mixer_reg_read(res, MXR_GRAPHIC_BASE(1));
+                        shadow = mixer_reg_read(res, MXR_GRAPHIC_BASE_S(1));
+                        if (base != shadow)
+                                goto out;
+                }
+
+                drm_handle_vblank(ctx->drm_dev, ctx->pipe);
+                exynos_drm_crtc_finish_pageflip(ctx->drm_dev, ctx->pipe);
+
+                /* set wait vsync event to zero and wake up queue. */
+                if (atomic_read(&ctx->wait_vsync_event)) {
+                        atomic_set(&ctx->wait_vsync_event, 0);
+                        wake_up(&ctx->wait_vsync_queue);
+                }
+        }
+
+out:
+        /* clear interrupts */
+        if (~val & MXR_INT_EN_VSYNC) {
+                /* vsync interrupt use different bit for read and clear */
+                val &= ~MXR_INT_EN_VSYNC;
+                val |= MXR_INT_CLEAR_VSYNC;
+        }
+        mixer_reg_write(res, MXR_INT_STATUS, val);
+
+        spin_unlock(&res->reg_slock);
+
+        return IRQ_HANDLED;
+}
+
+static int mixer_resources_init(struct mixer_context *mixer_ctx)
 {
-        struct exynos_drm_hdmi_context *drm_hdmi_ctx;
-        struct mixer_context *mdata = ctx;
-        struct drm_device *drm_dev;
+        struct device *dev = &mixer_ctx->pdev->dev;
+        struct mixer_resources *mixer_res = &mixer_ctx->mixer_res;
+        struct resource *res;
+        int ret;
 
-        drm_hdmi_ctx = mdata->parent_ctx;
-        drm_dev = drm_hdmi_ctx->drm_dev;
+        spin_lock_init(&mixer_res->reg_slock);
 
-        if (is_drm_iommu_supported(drm_dev)) {
-                if (enable)
-                        return drm_iommu_attach_device(drm_dev, mdata->dev);
+        mixer_res->mixer = devm_clk_get(dev, "mixer");
+        if (IS_ERR(mixer_res->mixer)) {
+                dev_err(dev, "failed to get clock 'mixer'\n");
+                return -ENODEV;
+        }
 
-                drm_iommu_detach_device(drm_dev, mdata->dev);
+        mixer_res->sclk_hdmi = devm_clk_get(dev, "sclk_hdmi");
+        if (IS_ERR(mixer_res->sclk_hdmi)) {
+                dev_err(dev, "failed to get clock 'sclk_hdmi'\n");
+                return -ENODEV;
+        }
+        res = platform_get_resource(mixer_ctx->pdev, IORESOURCE_MEM, 0);
+        if (res == NULL) {
+                dev_err(dev, "get memory resource failed.\n");
+                return -ENXIO;
         }
+
+        mixer_res->mixer_regs = devm_ioremap(dev, res->start,
+                                                        resource_size(res));
+        if (mixer_res->mixer_regs == NULL) {
+                dev_err(dev, "register mapping failed.\n");
+                return -ENXIO;
+        }
+
+        res = platform_get_resource(mixer_ctx->pdev, IORESOURCE_IRQ, 0);
+        if (res == NULL) {
+                dev_err(dev, "get interrupt resource failed.\n");
+                return -ENXIO;
+        }
+
+        ret = devm_request_irq(dev, res->start, mixer_irq_handler,
+                                                0, "drm_mixer", mixer_ctx);
+        if (ret) {
+                dev_err(dev, "request interrupt failed.\n");
+                return ret;
+        }
+        mixer_res->irq = res->start;
+
         return 0;
 }
 
-static int mixer_enable_vblank(void *ctx, int pipe)
+static int vp_resources_init(struct mixer_context *mixer_ctx)
 {
-        struct mixer_context *mixer_ctx = ctx;
-        struct mixer_resources *res = &mixer_ctx->mixer_res;
+        struct device *dev = &mixer_ctx->pdev->dev;
+        struct mixer_resources *mixer_res = &mixer_ctx->mixer_res;
+        struct resource *res;
+
+        mixer_res->vp = devm_clk_get(dev, "vp");
+        if (IS_ERR(mixer_res->vp)) {
+                dev_err(dev, "failed to get clock 'vp'\n");
+                return -ENODEV;
+        }
+        mixer_res->sclk_mixer = devm_clk_get(dev, "sclk_mixer");
+        if (IS_ERR(mixer_res->sclk_mixer)) {
+                dev_err(dev, "failed to get clock 'sclk_mixer'\n");
+                return -ENODEV;
+        }
+        mixer_res->sclk_dac = devm_clk_get(dev, "sclk_dac");
+        if (IS_ERR(mixer_res->sclk_dac)) {
+                dev_err(dev, "failed to get clock 'sclk_dac'\n");
+                return -ENODEV;
+        }
+
+        if (mixer_res->sclk_hdmi)
+                clk_set_parent(mixer_res->sclk_mixer, mixer_res->sclk_hdmi);
+
+        res = platform_get_resource(mixer_ctx->pdev, IORESOURCE_MEM, 1);
+        if (res == NULL) {
+                dev_err(dev, "get memory resource failed.\n");
+                return -ENXIO;
+        }
 
+        mixer_res->vp_regs = devm_ioremap(dev, res->start,
+                                                        resource_size(res));
+        if (mixer_res->vp_regs == NULL) {
+                dev_err(dev, "register mapping failed.\n");
+                return -ENXIO;
+        }
+
+        return 0;
+}
+
+static int mixer_initialize(struct exynos_drm_manager *mgr,
+                        struct drm_device *drm_dev, int pipe)
+{
+        int ret;
+        struct mixer_context *mixer_ctx = mgr->ctx;
+
+        mixer_ctx->drm_dev = drm_dev;
         mixer_ctx->pipe = pipe;
 
+        /* acquire resources: regs, irqs, clocks */
+        ret = mixer_resources_init(mixer_ctx);
+        if (ret) {
+                DRM_ERROR("mixer_resources_init failed ret=%d\n", ret);
+                return ret;
+        }
+
+        if (mixer_ctx->vp_enabled) {
+                /* acquire vp resources: regs, irqs, clocks */
+                ret = vp_resources_init(mixer_ctx);
+                if (ret) {
+                        DRM_ERROR("vp_resources_init failed ret=%d\n", ret);
+                        return ret;
+                }
+        }
+
+        if (!is_drm_iommu_supported(mixer_ctx->drm_dev))
+                return 0;
+
+        return drm_iommu_attach_device(mixer_ctx->drm_dev, mixer_ctx->dev);
+}
+
+static void mixer_mgr_remove(struct exynos_drm_manager *mgr)
+{
+        struct mixer_context *mixer_ctx = mgr->ctx;
+
+        if (is_drm_iommu_supported(mixer_ctx->drm_dev))
+                drm_iommu_detach_device(mixer_ctx->drm_dev, mixer_ctx->dev);
+}
+
+static int mixer_enable_vblank(struct exynos_drm_manager *mgr)
+{
+        struct mixer_context *mixer_ctx = mgr->ctx;
+        struct mixer_resources *res = &mixer_ctx->mixer_res;
+
+        if (!mixer_ctx->powered) {
+                mixer_ctx->int_en |= MXR_INT_EN_VSYNC;
+                return 0;
+        }
+
         /* enable vsync interrupt */
         mixer_reg_writemask(res, MXR_INT_EN, MXR_INT_EN_VSYNC,
                         MXR_INT_EN_VSYNC);
@@ -717,19 +885,19 @@ static int mixer_enable_vblank(void *ctx, int pipe)
         return 0;
 }
 
-static void mixer_disable_vblank(void *ctx)
+static void mixer_disable_vblank(struct exynos_drm_manager *mgr)
 {
-        struct mixer_context *mixer_ctx = ctx;
+        struct mixer_context *mixer_ctx = mgr->ctx;
         struct mixer_resources *res = &mixer_ctx->mixer_res;
 
         /* disable vsync interrupt */
         mixer_reg_writemask(res, MXR_INT_EN, 0, MXR_INT_EN_VSYNC);
 }
 
-static void mixer_win_mode_set(void *ctx,
-                              struct exynos_drm_overlay *overlay)
+static void mixer_win_mode_set(struct exynos_drm_manager *mgr,
+                        struct exynos_drm_overlay *overlay)
 {
-        struct mixer_context *mixer_ctx = ctx;
+        struct mixer_context *mixer_ctx = mgr->ctx;
         struct hdmi_win_data *win_data;
         int win;
 
@@ -778,9 +946,10 @@ static void mixer_win_mode_set(void *ctx,
         win_data->scan_flags = overlay->scan_flag;
 }
 
-static void mixer_win_commit(void *ctx, int win)
+static void mixer_win_commit(struct exynos_drm_manager *mgr, int zpos)
 {
-        struct mixer_context *mixer_ctx = ctx;
+        struct mixer_context *mixer_ctx = mgr->ctx;
+        int win = zpos == DEFAULT_ZPOS ? MIXER_DEFAULT_WIN : zpos;
 
         DRM_DEBUG_KMS("win: %d\n", win);
 
@@ -799,10 +968,11 @@ static void mixer_win_commit(void *ctx, int win)
         mixer_ctx->win_data[win].enabled = true;
 }
 
-static void mixer_win_disable(void *ctx, int win)
+static void mixer_win_disable(struct exynos_drm_manager *mgr, int zpos)
 {
-        struct mixer_context *mixer_ctx = ctx;
+        struct mixer_context *mixer_ctx = mgr->ctx;
         struct mixer_resources *res = &mixer_ctx->mixer_res;
+        int win = zpos == DEFAULT_ZPOS ? MIXER_DEFAULT_WIN : zpos;
         unsigned long flags;
 
         DRM_DEBUG_KMS("win: %d\n", win);
@@ -826,32 +996,9 @@ static void mixer_win_disable(void *ctx, int win)
         mixer_ctx->win_data[win].enabled = false;
 }
 
-static int mixer_check_mode(void *ctx, struct drm_display_mode *mode)
+static void mixer_wait_for_vblank(struct exynos_drm_manager *mgr)
 {
-        struct mixer_context *mixer_ctx = ctx;
-        u32 w, h;
-
-        w = mode->hdisplay;
-        h = mode->vdisplay;
-
-        DRM_DEBUG_KMS("xres=%d, yres=%d, refresh=%d, intl=%d\n",
-                mode->hdisplay, mode->vdisplay, mode->vrefresh,
-                (mode->flags & DRM_MODE_FLAG_INTERLACE) ? 1 : 0);
-
-        if (mixer_ctx->mxr_ver == MXR_VER_0_0_0_16 ||
-                mixer_ctx->mxr_ver == MXR_VER_128_0_0_184)
-                return 0;
-
-        if ((w >= 464 && w <= 720 && h >= 261 && h <= 576) ||
-                (w >= 1024 && w <= 1280 && h >= 576 && h <= 720) ||
-                (w >= 1664 && w <= 1920 && h >= 936 && h <= 1080))
-                return 0;
-
-        return -EINVAL;
-}
-static void mixer_wait_for_vblank(void *ctx)
-{
-        struct mixer_context *mixer_ctx = ctx;
+        struct mixer_context *mixer_ctx = mgr->ctx;
 
         mutex_lock(&mixer_ctx->mixer_mutex);
         if (!mixer_ctx->powered) {
@@ -872,21 +1019,23 @@ static void mixer_wait_for_vblank(void *ctx)
                 DRM_DEBUG_KMS("vblank wait timed out.\n");
 }
 
-static void mixer_window_suspend(struct mixer_context *ctx)
+static void mixer_window_suspend(struct exynos_drm_manager *mgr)
 {
+        struct mixer_context *ctx = mgr->ctx;
         struct hdmi_win_data *win_data;
         int i;
 
         for (i = 0; i < MIXER_WIN_NR; i++) {
                 win_data = &ctx->win_data[i];
                 win_data->resume = win_data->enabled;
-                mixer_win_disable(ctx, i);
+                mixer_win_disable(mgr, i);
         }
-        mixer_wait_for_vblank(ctx);
+        mixer_wait_for_vblank(mgr);
 }
 
-static void mixer_window_resume(struct mixer_context *ctx)
+static void mixer_window_resume(struct exynos_drm_manager *mgr)
 {
+        struct mixer_context *ctx = mgr->ctx;
         struct hdmi_win_data *win_data;
         int i;
 
@@ -894,11 +1043,14 @@ static void mixer_window_resume(struct mixer_context *ctx)
                 win_data = &ctx->win_data[i];
                 win_data->enabled = win_data->resume;
                 win_data->resume = false;
+                if (win_data->enabled)
+                        mixer_win_commit(mgr, i);
         }
 }
 
-static void mixer_poweron(struct mixer_context *ctx)
+static void mixer_poweron(struct exynos_drm_manager *mgr)
 {
+        struct mixer_context *ctx = mgr->ctx;
         struct mixer_resources *res = &ctx->mixer_res;
 
         mutex_lock(&ctx->mixer_mutex);
@@ -909,6 +1061,8 @@ static void mixer_poweron(struct mixer_context *ctx)
         ctx->powered = true;
         mutex_unlock(&ctx->mixer_mutex);
 
+        pm_runtime_get_sync(ctx->dev);
+
         clk_prepare_enable(res->mixer);
         if (ctx->vp_enabled) {
                 clk_prepare_enable(res->vp);
@@ -918,11 +1072,12 @@ static void mixer_poweron(struct mixer_context *ctx)
         mixer_reg_write(res, MXR_INT_EN, ctx->int_en);
         mixer_win_reset(ctx);
 
-        mixer_window_resume(ctx);
+        mixer_window_resume(mgr);
 }
 
-static void mixer_poweroff(struct mixer_context *ctx)
+static void mixer_poweroff(struct exynos_drm_manager *mgr)
 {
+        struct mixer_context *ctx = mgr->ctx;
         struct mixer_resources *res = &ctx->mixer_res;
 
         mutex_lock(&ctx->mixer_mutex);
@@ -930,7 +1085,7 @@ static void mixer_poweroff(struct mixer_context *ctx)
                 goto out;
         mutex_unlock(&ctx->mixer_mutex);
 
-        mixer_window_suspend(ctx);
+        mixer_window_suspend(mgr);
 
         ctx->int_en = mixer_reg_read(res, MXR_INT_EN);
 
@@ -940,6 +1095,8 @@ static void mixer_poweroff(struct mixer_context *ctx)
                 clk_disable_unprepare(res->sclk_mixer);
         }
 
+        pm_runtime_put_sync(ctx->dev);
+
         mutex_lock(&ctx->mixer_mutex);
         ctx->powered = false;
 
@@ -947,20 +1104,16 @@ out:
         mutex_unlock(&ctx->mixer_mutex);
 }
 
-static void mixer_dpms(void *ctx, int mode)
+static void mixer_dpms(struct exynos_drm_manager *mgr, int mode)
 {
-        struct mixer_context *mixer_ctx = ctx;
-
         switch (mode) {
         case DRM_MODE_DPMS_ON:
-                if (pm_runtime_suspended(mixer_ctx->dev))
-                        pm_runtime_get_sync(mixer_ctx->dev);
+                mixer_poweron(mgr);
                 break;
         case DRM_MODE_DPMS_STANDBY:
         case DRM_MODE_DPMS_SUSPEND:
         case DRM_MODE_DPMS_OFF:
-                if (!pm_runtime_suspended(mixer_ctx->dev))
-                        pm_runtime_put_sync(mixer_ctx->dev);
+                mixer_poweroff(mgr);
                 break;
         default:
                 DRM_DEBUG_KMS("unknown dpms mode: %d\n", mode);
@@ -968,169 +1121,42 @@ static void mixer_dpms(void *ctx, int mode)
         }
 }
 
-static struct exynos_mixer_ops mixer_ops = {
-        /* manager */
-        .iommu_on               = mixer_iommu_on,
-        .enable_vblank          = mixer_enable_vblank,
-        .disable_vblank         = mixer_disable_vblank,
-        .wait_for_vblank        = mixer_wait_for_vblank,
-        .dpms                   = mixer_dpms,
-
-        /* overlay */
-        .win_mode_set           = mixer_win_mode_set,
-        .win_commit             = mixer_win_commit,
-        .win_disable            = mixer_win_disable,
-
-        /* display */
-        .check_mode             = mixer_check_mode,
-};
-
-static irqreturn_t mixer_irq_handler(int irq, void *arg)
-{
-        struct exynos_drm_hdmi_context *drm_hdmi_ctx = arg;
-        struct mixer_context *ctx = drm_hdmi_ctx->ctx;
-        struct mixer_resources *res = &ctx->mixer_res;
-        u32 val, base, shadow;
-
-        spin_lock(&res->reg_slock);
-
-        /* read interrupt status for handling and clearing flags for VSYNC */
-        val = mixer_reg_read(res, MXR_INT_STATUS);
-
-        /* handling VSYNC */
-        if (val & MXR_INT_STATUS_VSYNC) {
-                /* interlace scan need to check shadow register */
-                if (ctx->interlace) {
-                        base = mixer_reg_read(res, MXR_GRAPHIC_BASE(0));
-                        shadow = mixer_reg_read(res, MXR_GRAPHIC_BASE_S(0));
-                        if (base != shadow)
-                                goto out;
-
-                        base = mixer_reg_read(res, MXR_GRAPHIC_BASE(1));
-                        shadow = mixer_reg_read(res, MXR_GRAPHIC_BASE_S(1));
-                        if (base != shadow)
-                                goto out;
-                }
-
-                drm_handle_vblank(drm_hdmi_ctx->drm_dev, ctx->pipe);
-                exynos_drm_crtc_finish_pageflip(drm_hdmi_ctx->drm_dev,
-                                ctx->pipe);
-
-                /* set wait vsync event to zero and wake up queue. */
-                if (atomic_read(&ctx->wait_vsync_event)) {
-                        atomic_set(&ctx->wait_vsync_event, 0);
-                        wake_up(&ctx->wait_vsync_queue);
-                }
-        }
-
-out:
-        /* clear interrupts */
-        if (~val & MXR_INT_EN_VSYNC) {
-                /* vsync interrupt use different bit for read and clear */
-                val &= ~MXR_INT_EN_VSYNC;
-                val |= MXR_INT_CLEAR_VSYNC;
-        }
-        mixer_reg_write(res, MXR_INT_STATUS, val);
-
-        spin_unlock(&res->reg_slock);
-
-        return IRQ_HANDLED;
-}
-
-static int mixer_resources_init(struct exynos_drm_hdmi_context *ctx,
-                                struct platform_device *pdev)
+/* Only valid for Mixer version 16.0.33.0 */
+int mixer_check_mode(struct drm_display_mode *mode)
 {
-        struct mixer_context *mixer_ctx = ctx->ctx;
-        struct device *dev = &pdev->dev;
-        struct mixer_resources *mixer_res = &mixer_ctx->mixer_res;
-        struct resource *res;
-        int ret;
-
-        spin_lock_init(&mixer_res->reg_slock);
-
-        mixer_res->mixer = devm_clk_get(dev, "mixer");
-        if (IS_ERR(mixer_res->mixer)) {
-                dev_err(dev, "failed to get clock 'mixer'\n");
-                return -ENODEV;
-        }
-
-        mixer_res->sclk_hdmi = devm_clk_get(dev, "sclk_hdmi");
-        if (IS_ERR(mixer_res->sclk_hdmi)) {
-                dev_err(dev, "failed to get clock 'sclk_hdmi'\n");
-                return -ENODEV;
-        }
-        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
-        if (res == NULL) {
-                dev_err(dev, "get memory resource failed.\n");
-                return -ENXIO;
-        }
+        u32 w, h;
 
-        mixer_res->mixer_regs = devm_ioremap(dev, res->start,
-                                                        resource_size(res));
-        if (mixer_res->mixer_regs == NULL) {
-                dev_err(dev, "register mapping failed.\n");
-                return -ENXIO;
-        }
+        w = mode->hdisplay;
+        h = mode->vdisplay;
 
-        res = platform_get_resource(pdev, IORESOURCE_IRQ, 0);
-        if (res == NULL) {
-                dev_err(dev, "get interrupt resource failed.\n");
-                return -ENXIO;
-        }
+        DRM_DEBUG_KMS("xres=%d, yres=%d, refresh=%d, intl=%d\n",
+                mode->hdisplay, mode->vdisplay, mode->vrefresh,
+                (mode->flags & DRM_MODE_FLAG_INTERLACE) ? 1 : 0);
 
-        ret = devm_request_irq(dev, res->start, mixer_irq_handler,
-                                                        0, "drm_mixer", ctx);
-        if (ret) {
-                dev_err(dev, "request interrupt failed.\n");
-                return ret;
-        }
-        mixer_res->irq = res->start;
+        if ((w >= 464 && w <= 720 && h >= 261 && h <= 576) ||
+                (w >= 1024 && w <= 1280 && h >= 576 && h <= 720) ||
+                (w >= 1664 && w <= 1920 && h >= 936 && h <= 1080))
+                return 0;
 
-        return 0;
+        return -EINVAL;
 }
 
-static int vp_resources_init(struct exynos_drm_hdmi_context *ctx,
-                             struct platform_device *pdev)
-{
-        struct mixer_context *mixer_ctx = ctx->ctx;
-        struct device *dev = &pdev->dev;
-        struct mixer_resources *mixer_res = &mixer_ctx->mixer_res;
-        struct resource *res;
-
-        mixer_res->vp = devm_clk_get(dev, "vp");
-        if (IS_ERR(mixer_res->vp)) {
-                dev_err(dev, "failed to get clock 'vp'\n");
-                return -ENODEV;
-        }
-        mixer_res->sclk_mixer = devm_clk_get(dev, "sclk_mixer");
-        if (IS_ERR(mixer_res->sclk_mixer)) {
-                dev_err(dev, "failed to get clock 'sclk_mixer'\n");
-                return -ENODEV;
-        }
-        mixer_res->sclk_dac = devm_clk_get(dev, "sclk_dac");
-        if (IS_ERR(mixer_res->sclk_dac)) {
-                dev_err(dev, "failed to get clock 'sclk_dac'\n");
-                return -ENODEV;
-        }
-
-        if (mixer_res->sclk_hdmi)
-                clk_set_parent(mixer_res->sclk_mixer, mixer_res->sclk_hdmi);
-
-        res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
-        if (res == NULL) {
-                dev_err(dev, "get memory resource failed.\n");
-                return -ENXIO;
-        }
-
-        mixer_res->vp_regs = devm_ioremap(dev, res->start,
-                                                        resource_size(res));
-        if (mixer_res->vp_regs == NULL) {
-                dev_err(dev, "register mapping failed.\n");
-                return -ENXIO;
-        }
+static struct exynos_drm_manager_ops mixer_manager_ops = {
+        .initialize             = mixer_initialize,
+        .remove                 = mixer_mgr_remove,
+        .dpms                   = mixer_dpms,
+        .enable_vblank          = mixer_enable_vblank,
+        .disable_vblank         = mixer_disable_vblank,
+        .wait_for_vblank        = mixer_wait_for_vblank,
+        .win_mode_set           = mixer_win_mode_set,
+        .win_commit             = mixer_win_commit,
+        .win_disable            = mixer_win_disable,
+};
 
-        return 0;
-}
+static struct exynos_drm_manager mixer_manager = {
+        .type                   = EXYNOS_DISPLAY_TYPE_HDMI,
+        .ops                    = &mixer_manager_ops,
+};
 
 static struct mixer_drv_data exynos5420_mxr_drv_data = {
         .version = MXR_VER_128_0_0_184,
@@ -1177,21 +1203,16 @@ static struct of_device_id mixer_match_types[] = {
 static int mixer_probe(struct platform_device *pdev)
 {
         struct device *dev = &pdev->dev;
-        struct exynos_drm_hdmi_context *drm_hdmi_ctx;
         struct mixer_context *ctx;
         struct mixer_drv_data *drv;
-        int ret;
 
         dev_info(dev, "probe start\n");
 
-        drm_hdmi_ctx = devm_kzalloc(dev, sizeof(*drm_hdmi_ctx),
-                                                                GFP_KERNEL);
-        if (!drm_hdmi_ctx)
-                return -ENOMEM;
-
-        ctx = devm_kzalloc(dev, sizeof(*ctx), GFP_KERNEL);
-        if (!ctx)
+        ctx = devm_kzalloc(&pdev->dev, sizeof(*ctx), GFP_KERNEL);
+        if (!ctx) {
+                DRM_ERROR("failed to alloc mixer context.\n");
                 return -ENOMEM;
+        }
 
         mutex_init(&ctx->mixer_mutex);
 
@@ -1204,46 +1225,20 @@ static int mixer_probe(struct platform_device *pdev)
                         platform_get_device_id(pdev)->driver_data;
         }
 
+        ctx->pdev = pdev;
         ctx->dev = dev;
-        ctx->parent_ctx = (void *)drm_hdmi_ctx;
-        drm_hdmi_ctx->ctx = (void *)ctx;
         ctx->vp_enabled = drv->is_vp_enabled;
         ctx->mxr_ver = drv->version;
         init_waitqueue_head(&ctx->wait_vsync_queue);
         atomic_set(&ctx->wait_vsync_event, 0);
 
-        platform_set_drvdata(pdev, drm_hdmi_ctx);
-
-        /* acquire resources: regs, irqs, clocks */
-        ret = mixer_resources_init(drm_hdmi_ctx, pdev);
-        if (ret) {
-                DRM_ERROR("mixer_resources_init failed\n");
-                goto fail;
-        }
-
-        if (ctx->vp_enabled) {
-                /* acquire vp resources: regs, irqs, clocks */
-                ret = vp_resources_init(drm_hdmi_ctx, pdev);
-                if (ret) {
-                        DRM_ERROR("vp_resources_init failed\n");
-                        goto fail;
-                }
-        }
-
-        /* attach mixer driver to common hdmi. */
-        exynos_mixer_drv_attach(drm_hdmi_ctx);
-
-        /* register specific callback point to common hdmi. */
-        exynos_mixer_ops_register(&mixer_ops);
+        mixer_manager.ctx = ctx;
+        platform_set_drvdata(pdev, &mixer_manager);
+        exynos_drm_manager_register(&mixer_manager);
 
         pm_runtime_enable(dev);
 
         return 0;
-
-
-fail:
-        dev_info(dev, "probe failed\n");
-        return ret;
 }
 
 static int mixer_remove(struct platform_device *pdev)
@@ -1255,70 +1250,10 @@ static int mixer_remove(struct platform_device *pdev)
         return 0;
 }
 
-#ifdef CONFIG_PM_SLEEP
-static int mixer_suspend(struct device *dev)
-{
-        struct exynos_drm_hdmi_context *drm_hdmi_ctx = get_mixer_context(dev);
-        struct mixer_context *ctx = drm_hdmi_ctx->ctx;
-
-        if (pm_runtime_suspended(dev)) {
-                DRM_DEBUG_KMS("Already suspended\n");
-                return 0;
-        }
-
-        mixer_poweroff(ctx);
-
-        return 0;
-}
-
-static int mixer_resume(struct device *dev)
-{
-        struct exynos_drm_hdmi_context *drm_hdmi_ctx = get_mixer_context(dev);
-        struct mixer_context *ctx = drm_hdmi_ctx->ctx;
-
-        if (!pm_runtime_suspended(dev)) {
-                DRM_DEBUG_KMS("Already resumed\n");
-                return 0;
-        }
-
-        mixer_poweron(ctx);
-
-        return 0;
-}
-#endif
-
-#ifdef CONFIG_PM_RUNTIME
-static int mixer_runtime_suspend(struct device *dev)
-{
-        struct exynos_drm_hdmi_context *drm_hdmi_ctx = get_mixer_context(dev);
-        struct mixer_context *ctx = drm_hdmi_ctx->ctx;
-
-        mixer_poweroff(ctx);
-
-        return 0;
-}
-
-static int mixer_runtime_resume(struct device *dev)
-{
-        struct exynos_drm_hdmi_context *drm_hdmi_ctx = get_mixer_context(dev);
-        struct mixer_context *ctx = drm_hdmi_ctx->ctx;
-
-        mixer_poweron(ctx);
-
-        return 0;
-}
-#endif
-
-static const struct dev_pm_ops mixer_pm_ops = {
-        SET_SYSTEM_SLEEP_PM_OPS(mixer_suspend, mixer_resume)
-        SET_RUNTIME_PM_OPS(mixer_runtime_suspend, mixer_runtime_resume, NULL)
-};
-
 struct platform_driver mixer_driver = {
         .driver = {
                 .name = "exynos-mixer",
                 .owner = THIS_MODULE,
-                .pm = &mixer_pm_ops,
                 .of_match_table = mixer_match_types,
         },
         .probe = mixer_probe,
diff --git a/drivers/gpu/drm/exynos/exynos_mixer.h b/drivers/gpu/drm/exynos/exynos_mixer.h
new file mode 100644
index 000000000000..3811e417f0e9
--- /dev/null
+++ b/drivers/gpu/drm/exynos/exynos_mixer.h
@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2013 Google, Inc.
+ *
+ * This software is licensed under the terms of the GNU General Public
+ * License version 2, as published by the Free Software Foundation, and
+ * may be copied, distributed, and modified under those terms.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#ifndef _EXYNOS_MIXER_H_
+#define _EXYNOS_MIXER_H_
+
+/* This function returns 0 if the given timing is valid for the mixer */
+int mixer_check_mode(struct drm_display_mode *mode);
+
+#endif
diff --git a/drivers/gpu/drm/gma500/Makefile b/drivers/gpu/drm/gma500/Makefile
index e9064dd9045d..b15315576376 100644
--- a/drivers/gpu/drm/gma500/Makefile
+++ b/drivers/gpu/drm/gma500/Makefile
@@ -13,9 +13,11 @@ gma500_gfx-y += \
           intel_i2c.o \
           intel_gmbus.o \
           mmu.o \
+          blitter.o \
           power.o \
           psb_drv.o \
           gma_display.o \
+          gma_device.o \
           psb_intel_display.o \
           psb_intel_lvds.o \
           psb_intel_modes.o \
diff --git a/drivers/gpu/drm/gma500/blitter.c b/drivers/gpu/drm/gma500/blitter.c
new file mode 100644
index 000000000000..9cd54a6fb899
--- /dev/null
+++ b/drivers/gpu/drm/gma500/blitter.c
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2014, Patrik Jakobsson
+ * All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * Authors: Patrik Jakobsson <patrik.r.jakobsson@gmail.com>
+ */
+
+#include "psb_drv.h"
+
+#include "blitter.h"
+#include "psb_reg.h"
+
+/* Wait for the blitter to be completely idle */
+int gma_blt_wait_idle(struct drm_psb_private *dev_priv)
+{
+        unsigned long stop = jiffies + HZ;
+        int busy = 1;
+
+        /* NOP for Cedarview */
+        if (IS_CDV(dev_priv->dev))
+                return 0;
+
+        /* First do a quick check */
+        if ((PSB_RSGX32(PSB_CR_2D_SOCIF) == _PSB_C2_SOCIF_EMPTY) &&
+            ((PSB_RSGX32(PSB_CR_2D_BLIT_STATUS) & _PSB_C2B_STATUS_BUSY) == 0))
+                return 0;
+
+        do {
+                busy = (PSB_RSGX32(PSB_CR_2D_SOCIF) != _PSB_C2_SOCIF_EMPTY);
+        } while (busy && !time_after_eq(jiffies, stop));
+
+        if (busy)
+                return -EBUSY;
+
+        do {
+                busy = ((PSB_RSGX32(PSB_CR_2D_BLIT_STATUS) &
+                        _PSB_C2B_STATUS_BUSY) != 0);
+        } while (busy && !time_after_eq(jiffies, stop));
+
+        /* If still busy, we probably have a hang */
+        return (busy) ? -EBUSY : 0;
+}
diff --git a/drivers/gpu/drm/gma500/blitter.h b/drivers/gpu/drm/gma500/blitter.h
new file mode 100644
index 000000000000..b83648df590d
--- /dev/null
+++ b/drivers/gpu/drm/gma500/blitter.h
@@ -0,0 +1,22 @@
+/*
+ * Copyright (c) 2014, Patrik Jakobsson
+ * All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * Authors: Patrik Jakobsson <patrik.r.jakobsson@gmail.com>
+ */
+
+#ifndef __BLITTER_H
+#define __BLITTER_H
+
+extern int gma_blt_wait_idle(struct drm_psb_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/gma500/cdv_device.c b/drivers/gpu/drm/gma500/cdv_device.c
index 5a9a6a3063a8..3531f90e53d0 100644
--- a/drivers/gpu/drm/gma500/cdv_device.c
+++ b/drivers/gpu/drm/gma500/cdv_device.c
@@ -26,6 +26,7 @@
 #include "psb_intel_reg.h"
 #include "intel_bios.h"
 #include "cdv_device.h"
+#include "gma_device.h"
 
 #define VGA_SR_INDEX            0x3c4
 #define VGA_SR_DATA             0x3c5
@@ -426,43 +427,6 @@ static int cdv_power_up(struct drm_device *dev)
         return 0;
 }
 
-/* FIXME ? - shared with Poulsbo */
-static void cdv_get_core_freq(struct drm_device *dev)
-{
-        uint32_t clock;
-        struct pci_dev *pci_root = pci_get_bus_and_slot(0, 0);
-        struct drm_psb_private *dev_priv = dev->dev_private;
-
-        pci_write_config_dword(pci_root, 0xD0, 0xD0050300);
-        pci_read_config_dword(pci_root, 0xD4, &clock);
-        pci_dev_put(pci_root);
-
-        switch (clock & 0x07) {
-        case 0:
-                dev_priv->core_freq = 100;
-                break;
-        case 1:
-                dev_priv->core_freq = 133;
-                break;
-        case 2:
-                dev_priv->core_freq = 150;
-                break;
-        case 3:
-                dev_priv->core_freq = 178;
-                break;
-        case 4:
-                dev_priv->core_freq = 200;
-                break;
-        case 5:
-        case 6:
-        case 7:
-                dev_priv->core_freq = 266;
-                break;
-        default:
-                dev_priv->core_freq = 0;
-        }
-}
-
 static void cdv_hotplug_work_func(struct work_struct *work)
 {
         struct drm_psb_private *dev_priv = container_of(work, struct drm_psb_private,
@@ -618,7 +582,7 @@ static int cdv_chip_setup(struct drm_device *dev)
         if (pci_enable_msi(dev->pdev))
                 dev_warn(dev->dev, "Enabling MSI failed!\n");
         dev_priv->regmap = cdv_regmap;
-        cdv_get_core_freq(dev);
+        gma_get_core_freq(dev);
         psb_intel_opregion_init(dev);
         psb_intel_init_bios(dev);
         cdv_hotplug_enable(dev, false);
diff --git a/drivers/gpu/drm/gma500/cdv_intel_crt.c b/drivers/gpu/drm/gma500/cdv_intel_crt.c
index 661af492173d..c18268cd516e 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_crt.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_crt.c
@@ -81,13 +81,6 @@ static int cdv_intel_crt_mode_valid(struct drm_connector *connector,
         return MODE_OK;
 }
 
-static bool cdv_intel_crt_mode_fixup(struct drm_encoder *encoder,
-                                 const struct drm_display_mode *mode,
-                                 struct drm_display_mode *adjusted_mode)
-{
-        return true;
-}
-
 static void cdv_intel_crt_mode_set(struct drm_encoder *encoder,
                                struct drm_display_mode *mode,
                                struct drm_display_mode *adjusted_mode)
@@ -224,7 +217,7 @@ static int cdv_intel_crt_set_property(struct drm_connector *connector,
 
 static const struct drm_encoder_helper_funcs cdv_intel_crt_helper_funcs = {
         .dpms = cdv_intel_crt_dpms,
-        .mode_fixup = cdv_intel_crt_mode_fixup,
+        .mode_fixup = gma_encoder_mode_fixup,
         .prepare = gma_encoder_prepare,
         .commit = gma_encoder_commit,
         .mode_set = cdv_intel_crt_mode_set,
diff --git a/drivers/gpu/drm/gma500/cdv_intel_display.c b/drivers/gpu/drm/gma500/cdv_intel_display.c
index 8fbfa06da62d..66727328832d 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_display.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_display.c
@@ -412,8 +412,11 @@ static bool cdv_intel_find_dp_pll(const struct gma_limit_t *limit,
                                   int refclk,
                                   struct gma_clock_t *best_clock)
 {
+        struct gma_crtc *gma_crtc = to_gma_crtc(crtc);
         struct gma_clock_t clock;
-        if (refclk == 27000) {
+
+        switch (refclk) {
+        case 27000:
                 if (target < 200000) {
                         clock.p1 = 2;
                         clock.p2 = 10;
@@ -427,7 +430,9 @@ static bool cdv_intel_find_dp_pll(const struct gma_limit_t *limit,
                         clock.m1 = 0;
                         clock.m2 = 98;
                 }
-        } else if (refclk == 100000) {
+                break;
+
+        case 100000:
                 if (target < 200000) {
                         clock.p1 = 2;
                         clock.p2 = 10;
@@ -441,12 +446,13 @@ static bool cdv_intel_find_dp_pll(const struct gma_limit_t *limit,
                         clock.m1 = 0;
                         clock.m2 = 133;
                 }
-        } else
+                break;
+
+        default:
                 return false;
-        clock.m = clock.m2 + 2;
-        clock.p = clock.p1 * clock.p2;
-        clock.vco = (refclk * clock.m) / clock.n;
-        clock.dot = clock.vco / clock.p;
+        }
+
+        gma_crtc->clock_funcs->clock(refclk, &clock);
         memcpy(best_clock, &clock, sizeof(struct gma_clock_t));
         return true;
 }
@@ -463,54 +469,11 @@ static bool cdv_intel_pipe_enabled(struct drm_device *dev, int pipe)
         crtc = dev_priv->pipe_to_crtc_mapping[pipe];
         gma_crtc = to_gma_crtc(crtc);
 
-        if (crtc->fb == NULL || !gma_crtc->active)
+        if (crtc->primary->fb == NULL || !gma_crtc->active)
                 return false;
         return true;
 }
 
-static bool cdv_intel_single_pipe_active (struct drm_device *dev)
-{
-        uint32_t pipe_enabled = 0;
-
-        if (cdv_intel_pipe_enabled(dev, 0))
-                pipe_enabled |= FIFO_PIPEA;
-
-        if (cdv_intel_pipe_enabled(dev, 1))
-                pipe_enabled |= FIFO_PIPEB;
-
-
-        DRM_DEBUG_KMS("pipe enabled %x\n", pipe_enabled);
-
-        if (pipe_enabled == FIFO_PIPEA || pipe_enabled == FIFO_PIPEB)
-                return true;
-        else
-                return false;
-}
-
-static bool is_pipeb_lvds(struct drm_device *dev, struct drm_crtc *crtc)
-{
-        struct gma_crtc *gma_crtc = to_gma_crtc(crtc);
-        struct drm_mode_config *mode_config = &dev->mode_config;
-        struct drm_connector *connector;
-
-        if (gma_crtc->pipe != 1)
-                return false;
-
-        list_for_each_entry(connector, &mode_config->connector_list, head) {
-                struct gma_encoder *gma_encoder =
-                                        gma_attached_encoder(connector);
-
-                if (!connector->encoder
-                    || connector->encoder->crtc != crtc)
-                        continue;
-
-                if (gma_encoder->type == INTEL_OUTPUT_LVDS)
-                        return true;
-        }
-
-        return false;
-}
-
 void cdv_disable_sr(struct drm_device *dev)
 {
         if (REG_READ(FW_BLC_SELF) & FW_BLC_SELF_EN) {
@@ -535,8 +498,10 @@ void cdv_disable_sr(struct drm_device *dev)
 void cdv_update_wm(struct drm_device *dev, struct drm_crtc *crtc)
 {
         struct drm_psb_private *dev_priv = dev->dev_private;
+        struct gma_crtc *gma_crtc = to_gma_crtc(crtc);
 
-        if (cdv_intel_single_pipe_active(dev)) {
+        /* Is only one pipe enabled? */
+        if (cdv_intel_pipe_enabled(dev, 0) ^ cdv_intel_pipe_enabled(dev, 1)) {
                 u32 fw;
 
                 fw = REG_READ(DSPFW1);
@@ -557,7 +522,9 @@ void cdv_update_wm(struct drm_device *dev, struct drm_crtc *crtc)
 
                 /* ignore FW4 */
 
-                if (is_pipeb_lvds(dev, crtc)) {
+                /* Is pipe b lvds ? */
+                if (gma_crtc->pipe == 1 &&
+                    gma_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
                         REG_WRITE(DSPFW5, 0x00040330);
                 } else {
                         fw = (3 << DSP_PLANE_B_FIFO_WM1_SHIFT) |
diff --git a/drivers/gpu/drm/gma500/cdv_intel_dp.c b/drivers/gpu/drm/gma500/cdv_intel_dp.c
index 0490ce36b53f..9ff30c2efadb 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_dp.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_dp.c
@@ -1693,7 +1693,7 @@ done:
                 struct drm_crtc *crtc = encoder->base.crtc;
                 drm_crtc_helper_set_mode(crtc, &crtc->mode,
                                          crtc->x, crtc->y,
-                                         crtc->fb);
+                                         crtc->primary->fb);
         }
 
         return 0;
diff --git a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
index 1c0d723b8d24..b99084b3f706 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
@@ -89,13 +89,6 @@ static void cdv_hdmi_mode_set(struct drm_encoder *encoder,
         REG_READ(hdmi_priv->hdmi_reg);
 }
 
-static bool cdv_hdmi_mode_fixup(struct drm_encoder *encoder,
-                                  const struct drm_display_mode *mode,
-                                  struct drm_display_mode *adjusted_mode)
-{
-        return true;
-}
-
 static void cdv_hdmi_dpms(struct drm_encoder *encoder, int mode)
 {
         struct drm_device *dev = encoder->dev;
@@ -199,7 +192,7 @@ static int cdv_hdmi_set_property(struct drm_connector *connector,
                     crtc->saved_mode.vdisplay != 0) {
                         if (centre) {
                                 if (!drm_crtc_helper_set_mode(encoder->crtc, &crtc->saved_mode,
-                                            encoder->crtc->x, encoder->crtc->y, encoder->crtc->fb))
+                                            encoder->crtc->x, encoder->crtc->y, encoder->crtc->primary->fb))
                                         return -1;
                         } else {
                                 struct drm_encoder_helper_funcs *helpers
@@ -262,7 +255,7 @@ static void cdv_hdmi_destroy(struct drm_connector *connector)
 
 static const struct drm_encoder_helper_funcs cdv_hdmi_helper_funcs = {
         .dpms = cdv_hdmi_dpms,
-        .mode_fixup = cdv_hdmi_mode_fixup,
+        .mode_fixup = gma_encoder_mode_fixup,
         .prepare = gma_encoder_prepare,
         .mode_set = cdv_hdmi_mode_set,
         .commit = gma_encoder_commit,
diff --git a/drivers/gpu/drm/gma500/cdv_intel_lvds.c b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
index 20e08e65d46c..8ecc920fc26d 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
@@ -494,7 +494,7 @@ static int cdv_intel_lvds_set_property(struct drm_connector *connector,
                                                       &crtc->saved_mode,
                                                       encoder->crtc->x,
                                                       encoder->crtc->y,
-                                                      encoder->crtc->fb))
+                                                      encoder->crtc->primary->fb))
                                 return -1;
                 }
         } else if (!strcmp(property->name, "backlight") && encoder) {
@@ -712,6 +712,7 @@ void cdv_intel_lvds_init(struct drm_device *dev,
          * Attempt to get the fixed panel mode from DDC.  Assume that the
          * preferred mode is the right one.
          */
+        mutex_lock(&dev->mode_config.mutex);
         psb_intel_ddc_get_modes(connector,
                                 &gma_encoder->ddc_bus->adapter);
         list_for_each_entry(scan, &connector->probed_modes, head) {
@@ -772,10 +773,12 @@ void cdv_intel_lvds_init(struct drm_device *dev,
         }
 
 out:
+        mutex_unlock(&dev->mode_config.mutex);
         drm_sysfs_connector_add(connector);
         return;
 
 failed_find:
+        mutex_unlock(&dev->mode_config.mutex);
         printk(KERN_ERR "Failed find\n");
         if (gma_encoder->ddc_bus)
                 psb_intel_i2c_destroy(gma_encoder->ddc_bus);
diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c
index 94b3fec22c28..e7fcc148f333 100644
--- a/drivers/gpu/drm/gma500/framebuffer.c
+++ b/drivers/gpu/drm/gma500/framebuffer.c
@@ -319,7 +319,7 @@ static struct gtt_range *psbfb_alloc(struct drm_device *dev, int aligned_size)
 {
         struct gtt_range *backing;
         /* Begin by trying to use stolen memory backing */
-        backing = psb_gtt_alloc_range(dev, aligned_size, "fb", 1);
+        backing = psb_gtt_alloc_range(dev, aligned_size, "fb", 1, PAGE_SIZE);
         if (backing) {
                 drm_gem_private_object_init(dev, &backing->gem, aligned_size);
                 return backing;
diff --git a/drivers/gpu/drm/gma500/gem.c b/drivers/gpu/drm/gma500/gem.c
index e2db48a81ed0..c707fa6fca85 100644
--- a/drivers/gpu/drm/gma500/gem.c
+++ b/drivers/gpu/drm/gma500/gem.c
@@ -62,9 +62,6 @@ int psb_gem_dumb_map_gtt(struct drm_file *file, struct drm_device *dev,
         int ret = 0;
         struct drm_gem_object *obj;
 
-        if (!(dev->driver->driver_features & DRIVER_GEM))
-                return -ENODEV;
-
         mutex_lock(&dev->struct_mutex);
 
         /* GEM does all our handle to object mapping */
@@ -98,8 +95,8 @@ unlock:
  *      it so that userspace can speak about it. This does the core work
  *      for the various methods that do/will create GEM objects for things
  */
-static int psb_gem_create(struct drm_file *file,
-        struct drm_device *dev, uint64_t size, uint32_t *handlep)
+int psb_gem_create(struct drm_file *file, struct drm_device *dev, u64 size,
+                   u32 *handlep, int stolen, u32 align)
 {
         struct gtt_range *r;
         int ret;
@@ -109,7 +106,7 @@ static int psb_gem_create(struct drm_file *file,
 
         /* Allocate our object - for now a direct gtt range which is not
            stolen memory backed */
-        r = psb_gtt_alloc_range(dev, size, "gem", 0);
+        r = psb_gtt_alloc_range(dev, size, "gem", 0, PAGE_SIZE);
         if (r == NULL) {
                 dev_err(dev->dev, "no memory for %lld byte GEM object\n", size);
                 return -ENOSPC;
@@ -153,7 +150,8 @@ int psb_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
 {
         args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
         args->size = args->pitch * args->height;
-        return psb_gem_create(file, dev, args->size, &args->handle);
+        return psb_gem_create(file, dev, args->size, &args->handle, 0,
+                              PAGE_SIZE);
 }
 
 /**
@@ -229,47 +227,3 @@ fail:
                 return VM_FAULT_SIGBUS;
         }
 }
-
-static int psb_gem_create_stolen(struct drm_file *file, struct drm_device *dev,
-                                                int size, u32 *handle)
-{
-        struct gtt_range *gtt = psb_gtt_alloc_range(dev, size, "gem", 1);
-        if (gtt == NULL)
-                return -ENOMEM;
-
-        drm_gem_private_object_init(dev, &gtt->gem, size);
-        if (drm_gem_handle_create(file, &gtt->gem, handle) == 0)
-                return 0;
-
-        drm_gem_object_release(&gtt->gem);
-        psb_gtt_free_range(dev, gtt);
-        return -ENOMEM;
-}
-
-/*
- *      GEM interfaces for our specific client
- */
-int psb_gem_create_ioctl(struct drm_device *dev, void *data,
-                                        struct drm_file *file)
-{
-        struct drm_psb_gem_create *args = data;
-        int ret;
-        if (args->flags & GMA_GEM_CREATE_STOLEN) {
-                ret = psb_gem_create_stolen(file, dev, args->size,
-                                                        &args->handle);
-                if (ret == 0)
-                        return 0;
-                /* Fall throguh */
-                args->flags &= ~GMA_GEM_CREATE_STOLEN;
-        }
-        return psb_gem_create(file, dev, args->size, &args->handle);
-}
-
-int psb_gem_mmap_ioctl(struct drm_device *dev, void *data,
-                                        struct drm_file *file)
-{
-        struct drm_psb_gem_mmap *args = data;
-        return dev->driver->dumb_map_offset(file, dev,
-                                                args->handle, &args->offset);
-}
-
diff --git a/drivers/gpu/drm/gma500/gem.h b/drivers/gpu/drm/gma500/gem.h
new file mode 100644
index 000000000000..1381c5190f46
--- /dev/null
+++ b/drivers/gpu/drm/gma500/gem.h
@@ -0,0 +1,21 @@
+/**************************************************************************
+ * Copyright (c) 2014 Patrik Jakobsson
+ * All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ **************************************************************************/
+
+#ifndef _GEM_H
+#define _GEM_H
+
+extern int psb_gem_create(struct drm_file *file, struct drm_device *dev,
+                          u64 size, u32 *handlep, int stolen, u32 align);
+#endif
diff --git a/drivers/gpu/drm/gma500/gma_device.c b/drivers/gpu/drm/gma500/gma_device.c
new file mode 100644
index 000000000000..4a295f9ba067
--- /dev/null
+++ b/drivers/gpu/drm/gma500/gma_device.c
@@ -0,0 +1,56 @@
+/**************************************************************************
+ * Copyright (c) 2011, Intel Corporation.
+ * All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ **************************************************************************/
+
+#include <drm/drmP.h>
+#include "psb_drv.h"
+
+void gma_get_core_freq(struct drm_device *dev)
+{
+        uint32_t clock;
+        struct pci_dev *pci_root = pci_get_bus_and_slot(0, 0);
+        struct drm_psb_private *dev_priv = dev->dev_private;
+
+        /*pci_write_config_dword(pci_root, 0xD4, 0x00C32004);*/
+        /*pci_write_config_dword(pci_root, 0xD0, 0xE0033000);*/
+
+        pci_write_config_dword(pci_root, 0xD0, 0xD0050300);
+        pci_read_config_dword(pci_root, 0xD4, &clock);
+        pci_dev_put(pci_root);
+
+        switch (clock & 0x07) {
+        case 0:
+                dev_priv->core_freq = 100;
+                break;
+        case 1:
+                dev_priv->core_freq = 133;
+                break;
+        case 2:
+                dev_priv->core_freq = 150;
+                break;
+        case 3:
+                dev_priv->core_freq = 178;
+                break;
+        case 4:
+                dev_priv->core_freq = 200;
+                break;
+        case 5:
+        case 6:
+        case 7:
+                dev_priv->core_freq = 266;
+                break;
+        default:
+                dev_priv->core_freq = 0;
+        }
+}
diff --git a/drivers/gpu/drm/gma500/gma_device.h b/drivers/gpu/drm/gma500/gma_device.h
new file mode 100644
index 000000000000..e1dbb007b820
--- /dev/null
+++ b/drivers/gpu/drm/gma500/gma_device.h
@@ -0,0 +1,21 @@
+/**************************************************************************
+ * Copyright (c) 2011, Intel Corporation.
+ * All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ **************************************************************************/
+
+#ifndef _GMA_DEVICE_H
+#define _GMA_DEVICE_H
+
+extern void gma_get_core_freq(struct drm_device *dev);
+
+#endif
diff --git a/drivers/gpu/drm/gma500/gma_display.c b/drivers/gpu/drm/gma500/gma_display.c
index 386de2c9dc86..9bb9bddd881a 100644
--- a/drivers/gpu/drm/gma500/gma_display.c
+++ b/drivers/gpu/drm/gma500/gma_display.c
@@ -59,7 +59,7 @@ int gma_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         struct drm_device *dev = crtc->dev;
         struct drm_psb_private *dev_priv = dev->dev_private;
         struct gma_crtc *gma_crtc = to_gma_crtc(crtc);
-        struct psb_framebuffer *psbfb = to_psb_fb(crtc->fb);
+        struct psb_framebuffer *psbfb = to_psb_fb(crtc->primary->fb);
         int pipe = gma_crtc->pipe;
         const struct psb_offset *map = &dev_priv->regmap[pipe];
         unsigned long start, offset;
@@ -70,7 +70,7 @@ int gma_pipe_set_base(struct drm_crtc *crtc, int x, int y,
                 return 0;
 
         /* no fb bound */
-        if (!crtc->fb) {
+        if (!crtc->primary->fb) {
                 dev_err(dev->dev, "No FB bound\n");
                 goto gma_pipe_cleaner;
         }
@@ -81,19 +81,19 @@ int gma_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         if (ret < 0)
                 goto gma_pipe_set_base_exit;
         start = psbfb->gtt->offset;
-        offset = y * crtc->fb->pitches[0] + x * (crtc->fb->bits_per_pixel / 8);
+        offset = y * crtc->primary->fb->pitches[0] + x * (crtc->primary->fb->bits_per_pixel / 8);
 
-        REG_WRITE(map->stride, crtc->fb->pitches[0]);
+        REG_WRITE(map->stride, crtc->primary->fb->pitches[0]);
 
         dspcntr = REG_READ(map->cntr);
         dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
 
-        switch (crtc->fb->bits_per_pixel) {
+        switch (crtc->primary->fb->bits_per_pixel) {
         case 8:
                 dspcntr |= DISPPLANE_8BPP;
                 break;
         case 16:
-                if (crtc->fb->depth == 15)
+                if (crtc->primary->fb->depth == 15)
                         dspcntr |= DISPPLANE_15_16BPP;
                 else
                         dspcntr |= DISPPLANE_16BPP;
@@ -485,6 +485,13 @@ int gma_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
         return 0;
 }
 
+bool gma_encoder_mode_fixup(struct drm_encoder *encoder,
+                            const struct drm_display_mode *mode,
+                            struct drm_display_mode *adjusted_mode)
+{
+        return true;
+}
+
 bool gma_crtc_mode_fixup(struct drm_crtc *crtc,
                          const struct drm_display_mode *mode,
                          struct drm_display_mode *adjusted_mode)
@@ -511,8 +518,8 @@ void gma_crtc_disable(struct drm_crtc *crtc)
 
         crtc_funcs->dpms(crtc, DRM_MODE_DPMS_OFF);
 
-        if (crtc->fb) {
-                gt = to_psb_fb(crtc->fb)->gtt;
+        if (crtc->primary->fb) {
+                gt = to_psb_fb(crtc->primary->fb)->gtt;
                 psb_gtt_unpin(gt);
         }
 }
diff --git a/drivers/gpu/drm/gma500/gma_display.h b/drivers/gpu/drm/gma500/gma_display.h
index 78b9f986a6e5..ed569d8a6af3 100644
--- a/drivers/gpu/drm/gma500/gma_display.h
+++ b/drivers/gpu/drm/gma500/gma_display.h
@@ -90,6 +90,9 @@ extern void gma_crtc_restore(struct drm_crtc *crtc);
 extern void gma_encoder_prepare(struct drm_encoder *encoder);
 extern void gma_encoder_commit(struct drm_encoder *encoder);
 extern void gma_encoder_destroy(struct drm_encoder *encoder);
+extern bool gma_encoder_mode_fixup(struct drm_encoder *encoder,
+                                   const struct drm_display_mode *mode,
+                                   struct drm_display_mode *adjusted_mode);
 
 /* Common clock related functions */
 extern const struct gma_limit_t *gma_limit(struct drm_crtc *crtc, int refclk);
diff --git a/drivers/gpu/drm/gma500/gtt.c b/drivers/gpu/drm/gma500/gtt.c
index 2db731f00930..592d205a0089 100644
--- a/drivers/gpu/drm/gma500/gtt.c
+++ b/drivers/gpu/drm/gma500/gtt.c
@@ -22,6 +22,7 @@
 #include <drm/drmP.h>
 #include <linux/shmem_fs.h>
 #include "psb_drv.h"
+#include "blitter.h"
 
 
 /*
@@ -105,11 +106,13 @@ static int psb_gtt_insert(struct drm_device *dev, struct gtt_range *r,
 
         /* Write our page entries into the GTT itself */
         for (i = r->roll; i < r->npage; i++) {
-                pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]), 0);
+                pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
+                                       PSB_MMU_CACHED_MEMORY);
                 iowrite32(pte, gtt_slot++);
         }
         for (i = 0; i < r->roll; i++) {
-                pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]), 0);
+                pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
+                                       PSB_MMU_CACHED_MEMORY);
                 iowrite32(pte, gtt_slot++);
         }
         /* Make sure all the entries are set before we return */
@@ -127,7 +130,7 @@ static int psb_gtt_insert(struct drm_device *dev, struct gtt_range *r,
  *      page table entries with the dummy page. This is protected via the gtt
  *      mutex which the caller must hold.
  */
-static void psb_gtt_remove(struct drm_device *dev, struct gtt_range *r)
+void psb_gtt_remove(struct drm_device *dev, struct gtt_range *r)
 {
         struct drm_psb_private *dev_priv = dev->dev_private;
         u32 __iomem *gtt_slot;
@@ -137,7 +140,8 @@ static void psb_gtt_remove(struct drm_device *dev, struct gtt_range *r)
         WARN_ON(r->stolen);
 
         gtt_slot = psb_gtt_entry(dev, r);
-        pte = psb_gtt_mask_pte(page_to_pfn(dev_priv->scratch_page), 0);
+        pte = psb_gtt_mask_pte(page_to_pfn(dev_priv->scratch_page),
+                               PSB_MMU_CACHED_MEMORY);
 
         for (i = 0; i < r->npage; i++)
                 iowrite32(pte, gtt_slot++);
@@ -176,11 +180,13 @@ void psb_gtt_roll(struct drm_device *dev, struct gtt_range *r, int roll)
         gtt_slot = psb_gtt_entry(dev, r);
 
         for (i = r->roll; i < r->npage; i++) {
-                pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]), 0);
+                pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
+                                       PSB_MMU_CACHED_MEMORY);
                 iowrite32(pte, gtt_slot++);
         }
         for (i = 0; i < r->roll; i++) {
-                pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]), 0);
+                pte = psb_gtt_mask_pte(page_to_pfn(r->pages[i]),
+                                       PSB_MMU_CACHED_MEMORY);
                 iowrite32(pte, gtt_slot++);
         }
         ioread32(gtt_slot - 1);
@@ -240,6 +246,7 @@ int psb_gtt_pin(struct gtt_range *gt)
         int ret = 0;
         struct drm_device *dev = gt->gem.dev;
         struct drm_psb_private *dev_priv = dev->dev_private;
+        u32 gpu_base = dev_priv->gtt.gatt_start;
 
         mutex_lock(&dev_priv->gtt_mutex);
 
@@ -252,6 +259,9 @@ int psb_gtt_pin(struct gtt_range *gt)
                         psb_gtt_detach_pages(gt);
                         goto out;
                 }
+                psb_mmu_insert_pages(psb_mmu_get_default_pd(dev_priv->mmu),
+                                     gt->pages, (gpu_base + gt->offset),
+                                     gt->npage, 0, 0, PSB_MMU_CACHED_MEMORY);
         }
         gt->in_gart++;
 out:
@@ -274,16 +284,30 @@ void psb_gtt_unpin(struct gtt_range *gt)
 {
         struct drm_device *dev = gt->gem.dev;
         struct drm_psb_private *dev_priv = dev->dev_private;
+        u32 gpu_base = dev_priv->gtt.gatt_start;
+        int ret;
 
+        /* While holding the gtt_mutex no new blits can be initiated */
         mutex_lock(&dev_priv->gtt_mutex);
 
+        /* Wait for any possible usage of the memory to be finished */
+        ret = gma_blt_wait_idle(dev_priv);
+        if (ret) {
+                DRM_ERROR("Failed to idle the blitter, unpin failed!");
+                goto out;
+        }
+
         WARN_ON(!gt->in_gart);
 
         gt->in_gart--;
         if (gt->in_gart == 0 && gt->stolen == 0) {
+                psb_mmu_remove_pages(psb_mmu_get_default_pd(dev_priv->mmu),
+                                     (gpu_base + gt->offset), gt->npage, 0, 0);
                 psb_gtt_remove(dev, gt);
                 psb_gtt_detach_pages(gt);
         }
+
+out:
         mutex_unlock(&dev_priv->gtt_mutex);
 }
 
@@ -306,7 +330,7 @@ void psb_gtt_unpin(struct gtt_range *gt)
  *      as in use.
  */
 struct gtt_range *psb_gtt_alloc_range(struct drm_device *dev, int len,
-                                                const char *name, int backed)
+                                      const char *name, int backed, u32 align)
 {
         struct drm_psb_private *dev_priv = dev->dev_private;
         struct gtt_range *gt;
@@ -334,7 +358,7 @@ struct gtt_range *psb_gtt_alloc_range(struct drm_device *dev, int len,
         /* Ensure this is set for non GEM objects */
         gt->gem.dev = dev;
         ret = allocate_resource(dev_priv->gtt_mem, &gt->resource,
-                                len, start, end, PAGE_SIZE, NULL, NULL);
+                                len, start, end, align, NULL, NULL);
         if (ret == 0) {
                 gt->offset = gt->resource.start - r->start;
                 return gt;
@@ -497,6 +521,7 @@ int psb_gtt_init(struct drm_device *dev, int resume)
         if (!resume)
                 dev_priv->vram_addr = ioremap_wc(dev_priv->stolen_base,
                                                  stolen_size);
+
         if (!dev_priv->vram_addr) {
                 dev_err(dev->dev, "Failure to map stolen base.\n");
                 ret = -ENOMEM;
@@ -512,7 +537,7 @@ int psb_gtt_init(struct drm_device *dev, int resume)
         dev_dbg(dev->dev, "Set up %d stolen pages starting at 0x%08x, GTT offset %dK\n",
                 num_pages, pfn_base << PAGE_SHIFT, 0);
         for (i = 0; i < num_pages; ++i) {
-                pte = psb_gtt_mask_pte(pfn_base + i, 0);
+                pte = psb_gtt_mask_pte(pfn_base + i, PSB_MMU_CACHED_MEMORY);
                 iowrite32(pte, dev_priv->gtt_map + i);
         }
 
@@ -521,7 +546,7 @@ int psb_gtt_init(struct drm_device *dev, int resume)
          */
 
         pfn_base = page_to_pfn(dev_priv->scratch_page);
-        pte = psb_gtt_mask_pte(pfn_base, 0);
+        pte = psb_gtt_mask_pte(pfn_base, PSB_MMU_CACHED_MEMORY);
         for (; i < gtt_pages; ++i)
                 iowrite32(pte, dev_priv->gtt_map + i);
 
diff --git a/drivers/gpu/drm/gma500/gtt.h b/drivers/gpu/drm/gma500/gtt.h
index 6191d10acf33..f5860a739bd8 100644
--- a/drivers/gpu/drm/gma500/gtt.h
+++ b/drivers/gpu/drm/gma500/gtt.h
@@ -53,7 +53,8 @@ struct gtt_range {
 };
 
 extern struct gtt_range *psb_gtt_alloc_range(struct drm_device *dev, int len,
-                                                const char *name, int backed);
+                                             const char *name, int backed,
+                                             u32 align);
 extern void psb_gtt_kref_put(struct gtt_range *gt);
 extern void psb_gtt_free_range(struct drm_device *dev, struct gtt_range *gt);
 extern int psb_gtt_pin(struct gtt_range *gt);
diff --git a/drivers/gpu/drm/gma500/mdfld_dsi_output.c b/drivers/gpu/drm/gma500/mdfld_dsi_output.c
index 860a4ee9baaf..6e91b20ce2e5 100644
--- a/drivers/gpu/drm/gma500/mdfld_dsi_output.c
+++ b/drivers/gpu/drm/gma500/mdfld_dsi_output.c
@@ -287,7 +287,7 @@ static int mdfld_dsi_connector_set_property(struct drm_connector *connector,
                                                 &gma_crtc->saved_mode,
                                                 encoder->crtc->x,
                                                 encoder->crtc->y,
-                                                encoder->crtc->fb))
+                                                encoder->crtc->primary->fb))
                                         goto set_prop_error;
                         } else {
                                 struct drm_encoder_helper_funcs *funcs =
diff --git a/drivers/gpu/drm/gma500/mdfld_intel_display.c b/drivers/gpu/drm/gma500/mdfld_intel_display.c
index 321c00a944e9..8cc8a5abbc7b 100644
--- a/drivers/gpu/drm/gma500/mdfld_intel_display.c
+++ b/drivers/gpu/drm/gma500/mdfld_intel_display.c
@@ -166,7 +166,7 @@ static int mdfld__intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         struct drm_device *dev = crtc->dev;
         struct drm_psb_private *dev_priv = dev->dev_private;
         struct gma_crtc *gma_crtc = to_gma_crtc(crtc);
-        struct psb_framebuffer *psbfb = to_psb_fb(crtc->fb);
+        struct psb_framebuffer *psbfb = to_psb_fb(crtc->primary->fb);
         int pipe = gma_crtc->pipe;
         const struct psb_offset *map = &dev_priv->regmap[pipe];
         unsigned long start, offset;
@@ -178,12 +178,12 @@ static int mdfld__intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         dev_dbg(dev->dev, "pipe = 0x%x.\n", pipe);
 
         /* no fb bound */
-        if (!crtc->fb) {
+        if (!crtc->primary->fb) {
                 dev_dbg(dev->dev, "No FB bound\n");
                 return 0;
         }
 
-        ret = check_fb(crtc->fb);
+        ret = check_fb(crtc->primary->fb);
         if (ret)
                 return ret;
 
@@ -196,18 +196,18 @@ static int mdfld__intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
                 return 0;
 
         start = psbfb->gtt->offset;
-        offset = y * crtc->fb->pitches[0] + x * (crtc->fb->bits_per_pixel / 8);
+        offset = y * crtc->primary->fb->pitches[0] + x * (crtc->primary->fb->bits_per_pixel / 8);
 
-        REG_WRITE(map->stride, crtc->fb->pitches[0]);
+        REG_WRITE(map->stride, crtc->primary->fb->pitches[0]);
         dspcntr = REG_READ(map->cntr);
         dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
 
-        switch (crtc->fb->bits_per_pixel) {
+        switch (crtc->primary->fb->bits_per_pixel) {
         case 8:
                 dspcntr |= DISPPLANE_8BPP;
                 break;
         case 16:
-                if (crtc->fb->depth == 15)
+                if (crtc->primary->fb->depth == 15)
                         dspcntr |= DISPPLANE_15_16BPP;
                 else
                         dspcntr |= DISPPLANE_16BPP;
@@ -700,7 +700,7 @@ static int mdfld_crtc_mode_set(struct drm_crtc *crtc,
         }
 #endif
 
-        ret = check_fb(crtc->fb);
+        ret = check_fb(crtc->primary->fb);
         if (ret)
                 return ret;
 
diff --git a/drivers/gpu/drm/gma500/mmu.c b/drivers/gpu/drm/gma500/mmu.c
index c3e67ba94446..0eaf11c19939 100644
--- a/drivers/gpu/drm/gma500/mmu.c
+++ b/drivers/gpu/drm/gma500/mmu.c
@@ -18,6 +18,7 @@
 #include <drm/drmP.h>
 #include "psb_drv.h"
 #include "psb_reg.h"
+#include "mmu.h"
 
 /*
  * Code for the SGX MMU:
@@ -47,51 +48,6 @@
  * but on average it should be fast.
  */
 
-struct psb_mmu_driver {
-        /* protects driver- and pd structures. Always take in read mode
-         * before taking the page table spinlock.
-         */
-        struct rw_semaphore sem;
-
-        /* protects page tables, directory tables and pt tables.
-         * and pt structures.
-         */
-        spinlock_t lock;
-
-        atomic_t needs_tlbflush;
-
-        uint8_t __iomem *register_map;
-        struct psb_mmu_pd *default_pd;
-        /*uint32_t bif_ctrl;*/
-        int has_clflush;
-        int clflush_add;
-        unsigned long clflush_mask;
-
-        struct drm_psb_private *dev_priv;
-};
-
-struct psb_mmu_pd;
-
-struct psb_mmu_pt {
-        struct psb_mmu_pd *pd;
-        uint32_t index;
-        uint32_t count;
-        struct page *p;
-        uint32_t *v;
-};
-
-struct psb_mmu_pd {
-        struct psb_mmu_driver *driver;
-        int hw_context;
-        struct psb_mmu_pt **tables;
-        struct page *p;
-        struct page *dummy_pt;
-        struct page *dummy_page;
-        uint32_t pd_mask;
-        uint32_t invalid_pde;
-        uint32_t invalid_pte;
-};
-
 static inline uint32_t psb_mmu_pt_index(uint32_t offset)
 {
         return (offset >> PSB_PTE_SHIFT) & 0x3FF;
@@ -102,13 +58,13 @@ static inline uint32_t psb_mmu_pd_index(uint32_t offset)
         return offset >> PSB_PDE_SHIFT;
 }
 
+#if defined(CONFIG_X86)
 static inline void psb_clflush(void *addr)
 {
         __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory");
 }
 
-static inline void psb_mmu_clflush(struct psb_mmu_driver *driver,
-                                   void *addr)
+static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
 {
         if (!driver->has_clflush)
                 return;
@@ -117,62 +73,77 @@ static inline void psb_mmu_clflush(struct psb_mmu_driver *driver,
         psb_clflush(addr);
         mb();
 }
+#else
 
-static void psb_page_clflush(struct psb_mmu_driver *driver, struct page* page)
-{
-        uint32_t clflush_add = driver->clflush_add >> PAGE_SHIFT;
-        uint32_t clflush_count = PAGE_SIZE / clflush_add;
-        int i;
-        uint8_t *clf;
-
-        clf = kmap_atomic(page);
-        mb();
-        for (i = 0; i < clflush_count; ++i) {
-                psb_clflush(clf);
-                clf += clflush_add;
-        }
-        mb();
-        kunmap_atomic(clf);
+static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
+{;
 }
 
-static void psb_pages_clflush(struct psb_mmu_driver *driver,
-                                struct page *page[], unsigned long num_pages)
-{
-        int i;
-
-        if (!driver->has_clflush)
-                return ;
+#endif
 
-        for (i = 0; i < num_pages; i++)
-                psb_page_clflush(driver, *page++);
-}
-
-static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver,
-                                    int force)
+static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)
 {
+        struct drm_device *dev = driver->dev;
+        struct drm_psb_private *dev_priv = dev->dev_private;
+
+        if (atomic_read(&driver->needs_tlbflush) || force) {
+                uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL);
+                PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
+
+                /* Make sure data cache is turned off before enabling it */
+                wmb();
+                PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
+                (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
+                if (driver->msvdx_mmu_invaldc)
+                        atomic_set(driver->msvdx_mmu_invaldc, 1);
+        }
         atomic_set(&driver->needs_tlbflush, 0);
 }
 
+#if 0
 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
 {
         down_write(&driver->sem);
         psb_mmu_flush_pd_locked(driver, force);
         up_write(&driver->sem);
 }
+#endif
 
-void psb_mmu_flush(struct psb_mmu_driver *driver, int rc_prot)
+void psb_mmu_flush(struct psb_mmu_driver *driver)
 {
-        if (rc_prot)
-                down_write(&driver->sem);
-        if (rc_prot)
-                up_write(&driver->sem);
+        struct drm_device *dev = driver->dev;
+        struct drm_psb_private *dev_priv = dev->dev_private;
+        uint32_t val;
+
+        down_write(&driver->sem);
+        val = PSB_RSGX32(PSB_CR_BIF_CTRL);
+        if (atomic_read(&driver->needs_tlbflush))
+                PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
+        else
+                PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL);
+
+        /* Make sure data cache is turned off and MMU is flushed before
+           restoring bank interface control register */
+        wmb();
+        PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC),
+                   PSB_CR_BIF_CTRL);
+        (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
+
+        atomic_set(&driver->needs_tlbflush, 0);
+        if (driver->msvdx_mmu_invaldc)
+                atomic_set(driver->msvdx_mmu_invaldc, 1);
+        up_write(&driver->sem);
 }
 
 void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)
 {
-        /*ttm_tt_cache_flush(&pd->p, 1);*/
-        psb_pages_clflush(pd->driver, &pd->p, 1);
+        struct drm_device *dev = pd->driver->dev;
+        struct drm_psb_private *dev_priv = dev->dev_private;
+        uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 :
+                          PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4;
+
         down_write(&pd->driver->sem);
+        PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset);
         wmb();
         psb_mmu_flush_pd_locked(pd->driver, 1);
         pd->hw_context = hw_context;
@@ -183,7 +154,6 @@ void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)
 static inline unsigned long psb_pd_addr_end(unsigned long addr,
                                             unsigned long end)
 {
-
         addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;
         return (addr < end) ? addr : end;
 }
@@ -223,12 +193,10 @@ struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
                 goto out_err3;
 
         if (!trap_pagefaults) {
-                pd->invalid_pde =
-                    psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),
-                                     invalid_type);
-                pd->invalid_pte =
-                    psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),
-                                     invalid_type);
+                pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),
+                                                   invalid_type);
+                pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),
+                                                   invalid_type);
         } else {
                 pd->invalid_pde = 0;
                 pd->invalid_pte = 0;
@@ -279,12 +247,16 @@ static void psb_mmu_free_pt(struct psb_mmu_pt *pt)
 void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)
 {
         struct psb_mmu_driver *driver = pd->driver;
+        struct drm_device *dev = driver->dev;
+        struct drm_psb_private *dev_priv = dev->dev_private;
         struct psb_mmu_pt *pt;
         int i;
 
         down_write(&driver->sem);
-        if (pd->hw_context != -1)
+        if (pd->hw_context != -1) {
+                PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4);
                 psb_mmu_flush_pd_locked(driver, 1);
+        }
 
         /* Should take the spinlock here, but we don't need to do that
            since we have the semaphore in write mode. */
@@ -331,7 +303,7 @@ static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
         for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
                 *ptes++ = pd->invalid_pte;
 
-
+#if defined(CONFIG_X86)
         if (pd->driver->has_clflush && pd->hw_context != -1) {
                 mb();
                 for (i = 0; i < clflush_count; ++i) {
@@ -340,7 +312,7 @@ static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
                 }
                 mb();
         }
-
+#endif
         kunmap_atomic(v);
         spin_unlock(lock);
 
@@ -351,7 +323,7 @@ static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
         return pt;
 }
 
-static struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
+struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
                                              unsigned long addr)
 {
         uint32_t index = psb_mmu_pd_index(addr);
@@ -383,7 +355,7 @@ static struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
                 kunmap_atomic((void *) v);
 
                 if (pd->hw_context != -1) {
-                        psb_mmu_clflush(pd->driver, (void *) &v[index]);
+                        psb_mmu_clflush(pd->driver, (void *)&v[index]);
                         atomic_set(&pd->driver->needs_tlbflush, 1);
                 }
         }
@@ -420,8 +392,7 @@ static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
                 pd->tables[pt->index] = NULL;
 
                 if (pd->hw_context != -1) {
-                        psb_mmu_clflush(pd->driver,
-                                        (void *) &v[pt->index]);
+                        psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);
                         atomic_set(&pd->driver->needs_tlbflush, 1);
                 }
                 kunmap_atomic(pt->v);
@@ -432,8 +403,8 @@ static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
         spin_unlock(&pd->driver->lock);
 }
 
-static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt,
-                                   unsigned long addr, uint32_t pte)
+static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,
+                                   uint32_t pte)
 {
         pt->v[psb_mmu_pt_index(addr)] = pte;
 }
@@ -444,69 +415,50 @@ static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,
         pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;
 }
 
-
-void psb_mmu_mirror_gtt(struct psb_mmu_pd *pd,
-                        uint32_t mmu_offset, uint32_t gtt_start,
-                        uint32_t gtt_pages)
+struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)
 {
-        uint32_t *v;
-        uint32_t start = psb_mmu_pd_index(mmu_offset);
-        struct psb_mmu_driver *driver = pd->driver;
-        int num_pages = gtt_pages;
+        struct psb_mmu_pd *pd;
 
         down_read(&driver->sem);
-        spin_lock(&driver->lock);
-
-        v = kmap_atomic(pd->p);
-        v += start;
-
-        while (gtt_pages--) {
-                *v++ = gtt_start | pd->pd_mask;
-                gtt_start += PAGE_SIZE;
-        }
-
-        /*ttm_tt_cache_flush(&pd->p, num_pages);*/
-        psb_pages_clflush(pd->driver, &pd->p, num_pages);
-        kunmap_atomic(v);
-        spin_unlock(&driver->lock);
-
-        if (pd->hw_context != -1)
-                atomic_set(&pd->driver->needs_tlbflush, 1);
+        pd = driver->default_pd;
+        up_read(&driver->sem);
 
-        up_read(&pd->driver->sem);
-        psb_mmu_flush_pd(pd->driver, 0);
+        return pd;
 }
 
-struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)
+/* Returns the physical address of the PD shared by sgx/msvdx */
+uint32_t psb_get_default_pd_addr(struct psb_mmu_driver *driver)
 {
         struct psb_mmu_pd *pd;
 
-        /* down_read(&driver->sem); */
-        pd = driver->default_pd;
-        /* up_read(&driver->sem); */
-
-        return pd;
+        pd = psb_mmu_get_default_pd(driver);
+        return page_to_pfn(pd->p) << PAGE_SHIFT;
 }
 
 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)
 {
+        struct drm_device *dev = driver->dev;
+        struct drm_psb_private *dev_priv = dev->dev_private;
+
+        PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL);
         psb_mmu_free_pagedir(driver->default_pd);
         kfree(driver);
 }
 
-struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers,
-                                        int trap_pagefaults,
-                                        int invalid_type,
-                                        struct drm_psb_private *dev_priv)
+struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev,
+                                           int trap_pagefaults,
+                                           int invalid_type,
+                                           atomic_t *msvdx_mmu_invaldc)
 {
         struct psb_mmu_driver *driver;
+        struct drm_psb_private *dev_priv = dev->dev_private;
 
         driver = kmalloc(sizeof(*driver), GFP_KERNEL);
 
         if (!driver)
                 return NULL;
-        driver->dev_priv = dev_priv;
 
+        driver->dev = dev;
         driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,
                                               invalid_type);
         if (!driver->default_pd)
@@ -515,17 +467,24 @@ struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers,
         spin_lock_init(&driver->lock);
         init_rwsem(&driver->sem);
         down_write(&driver->sem);
-        driver->register_map = registers;
         atomic_set(&driver->needs_tlbflush, 1);
+        driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;
+
+        driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL);
+        PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,
+                   PSB_CR_BIF_CTRL);
+        PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,
+                   PSB_CR_BIF_CTRL);
 
         driver->has_clflush = 0;
 
+#if defined(CONFIG_X86)
         if (boot_cpu_has(X86_FEATURE_CLFLUSH)) {
                 uint32_t tfms, misc, cap0, cap4, clflush_size;
 
                 /*
-                 * clflush size is determined at kernel setup for x86_64
-                 *  but not for i386. We have to do it here.
+                 * clflush size is determined at kernel setup for x86_64 but not
+                 * for i386. We have to do it here.
                  */
 
                 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);
@@ -536,6 +495,7 @@ struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers,
                 driver->clflush_mask = driver->clflush_add - 1;
                 driver->clflush_mask = ~driver->clflush_mask;
         }
+#endif
 
         up_write(&driver->sem);
         return driver;
@@ -545,9 +505,9 @@ out_err1:
         return NULL;
 }
 
-static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd,
-                               unsigned long address, uint32_t num_pages,
-                               uint32_t desired_tile_stride,
+#if defined(CONFIG_X86)
+static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
+                               uint32_t num_pages, uint32_t desired_tile_stride,
                                uint32_t hw_tile_stride)
 {
         struct psb_mmu_pt *pt;
@@ -561,11 +521,8 @@ static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd,
         unsigned long clflush_add = pd->driver->clflush_add;
         unsigned long clflush_mask = pd->driver->clflush_mask;
 
-        if (!pd->driver->has_clflush) {
-                /*ttm_tt_cache_flush(&pd->p, num_pages);*/
-                psb_pages_clflush(pd->driver, &pd->p, num_pages);
+        if (!pd->driver->has_clflush)
                 return;
-        }
 
         if (hw_tile_stride)
                 rows = num_pages / desired_tile_stride;
@@ -586,10 +543,8 @@ static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd,
                         if (!pt)
                                 continue;
                         do {
-                                psb_clflush(&pt->v
-                                            [psb_mmu_pt_index(addr)]);
-                        } while (addr +=
-                                 clflush_add,
+                                psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);
+                        } while (addr += clflush_add,
                                  (addr & clflush_mask) < next);
 
                         psb_mmu_pt_unmap_unlock(pt);
@@ -598,6 +553,14 @@ static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd,
         }
         mb();
 }
+#else
+static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
+                               uint32_t num_pages, uint32_t desired_tile_stride,
+                               uint32_t hw_tile_stride)
+{
+        drm_ttm_cache_flush();
+}
+#endif
 
 void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
                                  unsigned long address, uint32_t num_pages)
@@ -633,7 +596,7 @@ out:
         up_read(&pd->driver->sem);
 
         if (pd->hw_context != -1)
-                psb_mmu_flush(pd->driver, 0);
+                psb_mmu_flush(pd->driver);
 
         return;
 }
@@ -660,7 +623,7 @@ void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,
         add = desired_tile_stride << PAGE_SHIFT;
         row_add = hw_tile_stride << PAGE_SHIFT;
 
-        /* down_read(&pd->driver->sem); */
+        down_read(&pd->driver->sem);
 
         /* Make sure we only need to flush this processor's cache */
 
@@ -688,10 +651,10 @@ void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,
                 psb_mmu_flush_ptes(pd, f_address, num_pages,
                                    desired_tile_stride, hw_tile_stride);
 
-        /* up_read(&pd->driver->sem); */
+        up_read(&pd->driver->sem);
 
         if (pd->hw_context != -1)
-                psb_mmu_flush(pd->driver, 0);
+                psb_mmu_flush(pd->driver);
 }
 
 int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
@@ -704,7 +667,7 @@ int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
         unsigned long end;
         unsigned long next;
         unsigned long f_address = address;
-        int ret = 0;
+        int ret = -ENOMEM;
 
         down_read(&pd->driver->sem);
 
@@ -726,6 +689,7 @@ int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
                 psb_mmu_pt_unmap_unlock(pt);
 
         } while (addr = next, next != end);
+        ret = 0;
 
 out:
         if (pd->hw_context != -1)
@@ -734,15 +698,15 @@ out:
         up_read(&pd->driver->sem);
 
         if (pd->hw_context != -1)
-                psb_mmu_flush(pd->driver, 1);
+                psb_mmu_flush(pd->driver);
 
-        return ret;
+        return 0;
 }
 
 int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
                          unsigned long address, uint32_t num_pages,
-                         uint32_t desired_tile_stride,
-                         uint32_t hw_tile_stride, int type)
+                         uint32_t desired_tile_stride, uint32_t hw_tile_stride,
+                         int type)
 {
         struct psb_mmu_pt *pt;
         uint32_t rows = 1;
@@ -754,7 +718,7 @@ int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
         unsigned long add;
         unsigned long row_add;
         unsigned long f_address = address;
-        int ret = 0;
+        int ret = -ENOMEM;
 
         if (hw_tile_stride) {
                 if (num_pages % desired_tile_stride != 0)
@@ -777,14 +741,11 @@ int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
                 do {
                         next = psb_pd_addr_end(addr, end);
                         pt = psb_mmu_pt_alloc_map_lock(pd, addr);
-                        if (!pt) {
-                                ret = -ENOMEM;
+                        if (!pt)
                                 goto out;
-                        }
                         do {
-                                pte =
-                                    psb_mmu_mask_pte(page_to_pfn(*pages++),
-                                                     type);
+                                pte = psb_mmu_mask_pte(page_to_pfn(*pages++),
+                                                       type);
                                 psb_mmu_set_pte(pt, addr, pte);
                                 pt->count++;
                         } while (addr += PAGE_SIZE, addr < next);
@@ -794,6 +755,8 @@ int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
 
                 address += row_add;
         }
+
+        ret = 0;
 out:
         if (pd->hw_context != -1)
                 psb_mmu_flush_ptes(pd, f_address, num_pages,
@@ -802,7 +765,7 @@ out:
         up_read(&pd->driver->sem);
 
         if (pd->hw_context != -1)
-                psb_mmu_flush(pd->driver, 1);
+                psb_mmu_flush(pd->driver);
 
         return ret;
 }
diff --git a/drivers/gpu/drm/gma500/mmu.h b/drivers/gpu/drm/gma500/mmu.h
new file mode 100644
index 000000000000..e89abec6209d
--- /dev/null
+++ b/drivers/gpu/drm/gma500/mmu.h
@@ -0,0 +1,93 @@
+/**************************************************************************
+ * Copyright (c) 2007-2011, Intel Corporation.
+ * All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms and conditions of the GNU General Public License,
+ * version 2, as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ **************************************************************************/
+
+#ifndef __MMU_H
+#define __MMU_H
+
+struct psb_mmu_driver {
+        /* protects driver- and pd structures. Always take in read mode
+         * before taking the page table spinlock.
+         */
+        struct rw_semaphore sem;
+
+        /* protects page tables, directory tables and pt tables.
+         * and pt structures.
+         */
+        spinlock_t lock;
+
+        atomic_t needs_tlbflush;
+        atomic_t *msvdx_mmu_invaldc;
+        struct psb_mmu_pd *default_pd;
+        uint32_t bif_ctrl;
+        int has_clflush;
+        int clflush_add;
+        unsigned long clflush_mask;
+
+        struct drm_device *dev;
+};
+
+struct psb_mmu_pd;
+
+struct psb_mmu_pt {
+        struct psb_mmu_pd *pd;
+        uint32_t index;
+        uint32_t count;
+        struct page *p;
+        uint32_t *v;
+};
+
+struct psb_mmu_pd {
+        struct psb_mmu_driver *driver;
+        int hw_context;
+        struct psb_mmu_pt **tables;
+        struct page *p;
+        struct page *dummy_pt;
+        struct page *dummy_page;
+        uint32_t pd_mask;
+        uint32_t invalid_pde;
+        uint32_t invalid_pte;
+};
+
+extern struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev,
+                                                  int trap_pagefaults,
+                                                  int invalid_type,
+                                                  atomic_t *msvdx_mmu_invaldc);
+extern void psb_mmu_driver_takedown(struct psb_mmu_driver *driver);
+extern struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver
+                                                 *driver);
+extern struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
+                                           int trap_pagefaults,
+                                           int invalid_type);
+extern void psb_mmu_free_pagedir(struct psb_mmu_pd *pd);
+extern void psb_mmu_flush(struct psb_mmu_driver *driver);
+extern void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
+                                        unsigned long address,
+                                        uint32_t num_pages);
+extern int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd,
+                                       uint32_t start_pfn,
+                                       unsigned long address,
+                                       uint32_t num_pages, int type);
+extern int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,
+                                  unsigned long *pfn);
+extern void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context);
+extern int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
+                                unsigned long address, uint32_t num_pages,
+                                uint32_t desired_tile_stride,
+                                uint32_t hw_tile_stride, int type);
+extern void psb_mmu_remove_pages(struct psb_mmu_pd *pd,
+                                 unsigned long address, uint32_t num_pages,
+                                 uint32_t desired_tile_stride,
+                                 uint32_t hw_tile_stride);
+
+#endif
diff --git a/drivers/gpu/drm/gma500/oaktrail_crtc.c b/drivers/gpu/drm/gma500/oaktrail_crtc.c
index 8195e8592107..2de216c2374f 100644
--- a/drivers/gpu/drm/gma500/oaktrail_crtc.c
+++ b/drivers/gpu/drm/gma500/oaktrail_crtc.c
@@ -599,7 +599,7 @@ static int oaktrail_pipe_set_base(struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         struct drm_psb_private *dev_priv = dev->dev_private;
         struct gma_crtc *gma_crtc = to_gma_crtc(crtc);
-        struct psb_framebuffer *psbfb = to_psb_fb(crtc->fb);
+        struct psb_framebuffer *psbfb = to_psb_fb(crtc->primary->fb);
         int pipe = gma_crtc->pipe;
         const struct psb_offset *map = &dev_priv->regmap[pipe];
         unsigned long start, offset;
@@ -608,7 +608,7 @@ static int oaktrail_pipe_set_base(struct drm_crtc *crtc,
         int ret = 0;
 
         /* no fb bound */
-        if (!crtc->fb) {
+        if (!crtc->primary->fb) {
                 dev_dbg(dev->dev, "No FB bound\n");
                 return 0;
         }
@@ -617,19 +617,19 @@ static int oaktrail_pipe_set_base(struct drm_crtc *crtc,
                 return 0;
 
         start = psbfb->gtt->offset;
-        offset = y * crtc->fb->pitches[0] + x * (crtc->fb->bits_per_pixel / 8);
+        offset = y * crtc->primary->fb->pitches[0] + x * (crtc->primary->fb->bits_per_pixel / 8);
 
-        REG_WRITE(map->stride, crtc->fb->pitches[0]);
+        REG_WRITE(map->stride, crtc->primary->fb->pitches[0]);
 
         dspcntr = REG_READ(map->cntr);
         dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
 
-        switch (crtc->fb->bits_per_pixel) {
+        switch (crtc->primary->fb->bits_per_pixel) {
         case 8:
                 dspcntr |= DISPPLANE_8BPP;
                 break;
         case 16:
-                if (crtc->fb->depth == 15)
+                if (crtc->primary->fb->depth == 15)
                         dspcntr |= DISPPLANE_15_16BPP;
                 else
                         dspcntr |= DISPPLANE_16BPP;
diff --git a/drivers/gpu/drm/gma500/oaktrail_hdmi.c b/drivers/gpu/drm/gma500/oaktrail_hdmi.c
index 38153143ed8c..cf018ddcc5a6 100644
--- a/drivers/gpu/drm/gma500/oaktrail_hdmi.c
+++ b/drivers/gpu/drm/gma500/oaktrail_hdmi.c
@@ -523,13 +523,6 @@ static int oaktrail_hdmi_mode_valid(struct drm_connector *connector,
         return MODE_OK;
 }
 
-static bool oaktrail_hdmi_mode_fixup(struct drm_encoder *encoder,
-                                 const struct drm_display_mode *mode,
-                                 struct drm_display_mode *adjusted_mode)
-{
-        return true;
-}
-
 static enum drm_connector_status
 oaktrail_hdmi_detect(struct drm_connector *connector, bool force)
 {
@@ -608,7 +601,7 @@ static void oaktrail_hdmi_destroy(struct drm_connector *connector)
 
 static const struct drm_encoder_helper_funcs oaktrail_hdmi_helper_funcs = {
         .dpms = oaktrail_hdmi_dpms,
-        .mode_fixup = oaktrail_hdmi_mode_fixup,
+        .mode_fixup = gma_encoder_mode_fixup,
         .prepare = gma_encoder_prepare,
         .mode_set = oaktrail_hdmi_mode_set,
         .commit = gma_encoder_commit,
diff --git a/drivers/gpu/drm/gma500/oaktrail_lvds.c b/drivers/gpu/drm/gma500/oaktrail_lvds.c
index 5e0697862736..9b099468a5db 100644
--- a/drivers/gpu/drm/gma500/oaktrail_lvds.c
+++ b/drivers/gpu/drm/gma500/oaktrail_lvds.c
@@ -359,6 +359,7 @@ void oaktrail_lvds_init(struct drm_device *dev,
          *    if closed, act like it's not there for now
          */
 
+        mutex_lock(&dev->mode_config.mutex);
         i2c_adap = i2c_get_adapter(dev_priv->ops->i2c_bus);
         if (i2c_adap == NULL)
                 dev_err(dev->dev, "No ddc adapter available!\n");
@@ -401,10 +402,14 @@ void oaktrail_lvds_init(struct drm_device *dev,
         }
 
 out:
+        mutex_unlock(&dev->mode_config.mutex);
+
         drm_sysfs_connector_add(connector);
         return;
 
 failed_find:
+        mutex_unlock(&dev->mode_config.mutex);
+
         dev_dbg(dev->dev, "No LVDS modes found, disabling.\n");
         if (gma_encoder->ddc_bus)
                 psb_intel_i2c_destroy(gma_encoder->ddc_bus);
diff --git a/drivers/gpu/drm/gma500/opregion.c b/drivers/gpu/drm/gma500/opregion.c
index 13ec6283bf59..ab696ca7eeec 100644
--- a/drivers/gpu/drm/gma500/opregion.c
+++ b/drivers/gpu/drm/gma500/opregion.c
@@ -173,10 +173,13 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
         return 0;
 }
 
-void psb_intel_opregion_asle_intr(struct drm_device *dev)
+static void psb_intel_opregion_asle_work(struct work_struct *work)
 {
-        struct drm_psb_private *dev_priv = dev->dev_private;
-        struct opregion_asle *asle = dev_priv->opregion.asle;
+        struct psb_intel_opregion *opregion =
+                container_of(work, struct psb_intel_opregion, asle_work);
+        struct drm_psb_private *dev_priv =
+                container_of(opregion, struct drm_psb_private, opregion);
+        struct opregion_asle *asle = opregion->asle;
         u32 asle_stat = 0;
         u32 asle_req;
 
@@ -190,9 +193,18 @@ void psb_intel_opregion_asle_intr(struct drm_device *dev)
         }
 
         if (asle_req & ASLE_SET_BACKLIGHT)
-                asle_stat |= asle_set_backlight(dev, asle->bclp);
+                asle_stat |= asle_set_backlight(dev_priv->dev, asle->bclp);
 
         asle->aslc = asle_stat;
+
+}
+
+void psb_intel_opregion_asle_intr(struct drm_device *dev)
+{
+        struct drm_psb_private *dev_priv = dev->dev_private;
+
+        if (dev_priv->opregion.asle)
+                schedule_work(&dev_priv->opregion.asle_work);
 }
 
 #define ASLE_ALS_EN    (1<<0)
@@ -282,6 +294,8 @@ void psb_intel_opregion_fini(struct drm_device *dev)
                 unregister_acpi_notifier(&psb_intel_opregion_notifier);
         }
 
+        cancel_work_sync(&opregion->asle_work);
+
         /* just clear all opregion memory pointers now */
         iounmap(opregion->header);
         opregion->header = NULL;
@@ -304,6 +318,9 @@ int psb_intel_opregion_setup(struct drm_device *dev)
                 DRM_DEBUG_DRIVER("ACPI Opregion not supported\n");
                 return -ENOTSUPP;
         }
+
+        INIT_WORK(&opregion->asle_work, psb_intel_opregion_asle_work);
+
         DRM_DEBUG("OpRegion detected at 0x%8x\n", opregion_phy);
         base = acpi_os_ioremap(opregion_phy, 8*1024);
         if (!base)
diff --git a/drivers/gpu/drm/gma500/psb_device.c b/drivers/gpu/drm/gma500/psb_device.c
index 23fb33f1471b..07df7d4eea72 100644
--- a/drivers/gpu/drm/gma500/psb_device.c
+++ b/drivers/gpu/drm/gma500/psb_device.c
@@ -26,6 +26,7 @@
 #include "psb_intel_reg.h"
 #include "intel_bios.h"
 #include "psb_device.h"
+#include "gma_device.h"
 
 static int psb_output_init(struct drm_device *dev)
 {
@@ -257,45 +258,6 @@ static int psb_power_up(struct drm_device *dev)
         return 0;
 }
 
-static void psb_get_core_freq(struct drm_device *dev)
-{
-        uint32_t clock;
-        struct pci_dev *pci_root = pci_get_bus_and_slot(0, 0);
-        struct drm_psb_private *dev_priv = dev->dev_private;
-
-        /*pci_write_config_dword(pci_root, 0xD4, 0x00C32004);*/
-        /*pci_write_config_dword(pci_root, 0xD0, 0xE0033000);*/
-
-        pci_write_config_dword(pci_root, 0xD0, 0xD0050300);
-        pci_read_config_dword(pci_root, 0xD4, &clock);
-        pci_dev_put(pci_root);
-
-        switch (clock & 0x07) {
-        case 0:
-                dev_priv->core_freq = 100;
-                break;
-        case 1:
-                dev_priv->core_freq = 133;
-                break;
-        case 2:
-                dev_priv->core_freq = 150;
-                break;
-        case 3:
-                dev_priv->core_freq = 178;
-                break;
-        case 4:
-                dev_priv->core_freq = 200;
-                break;
-        case 5:
-        case 6:
-        case 7:
-                dev_priv->core_freq = 266;
-                break;
-        default:
-                dev_priv->core_freq = 0;
-        }
-}
-
 /* Poulsbo */
 static const struct psb_offset psb_regmap[2] = {
         {
@@ -352,7 +314,7 @@ static int psb_chip_setup(struct drm_device *dev)
 {
         struct drm_psb_private *dev_priv = dev->dev_private;
         dev_priv->regmap = psb_regmap;
-        psb_get_core_freq(dev);
+        gma_get_core_freq(dev);
         gma_intel_setup_gmbus(dev);
         psb_intel_opregion_init(dev);
         psb_intel_init_bios(dev);
diff --git a/drivers/gpu/drm/gma500/psb_drv.c b/drivers/gpu/drm/gma500/psb_drv.c
index 1199180667c9..b686e56646eb 100644
--- a/drivers/gpu/drm/gma500/psb_drv.c
+++ b/drivers/gpu/drm/gma500/psb_drv.c
@@ -21,7 +21,6 @@
 
 #include <drm/drmP.h>
 #include <drm/drm.h>
-#include <drm/gma_drm.h>
 #include "psb_drv.h"
 #include "framebuffer.h"
 #include "psb_reg.h"
@@ -37,56 +36,65 @@
 #include <acpi/video.h>
 #include <linux/module.h>
 
-static int drm_psb_trap_pagefaults;
-
-static int psb_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
-
-MODULE_PARM_DESC(trap_pagefaults, "Error and reset on MMU pagefaults");
-module_param_named(trap_pagefaults, drm_psb_trap_pagefaults, int, 0600);
-
+static struct drm_driver driver;
+static int psb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
 
+/*
+ * The table below contains a mapping of the PCI vendor ID and the PCI Device ID
+ * to the different groups of PowerVR 5-series chip designs
+ *
+ * 0x8086 = Intel Corporation
+ *
+ * PowerVR SGX535    - Poulsbo    - Intel GMA 500, Intel Atom Z5xx
+ * PowerVR SGX535    - Moorestown - Intel GMA 600
+ * PowerVR SGX535    - Oaktrail   - Intel GMA 600, Intel Atom Z6xx, E6xx
+ * PowerVR SGX540    - Medfield   - Intel Atom Z2460
+ * PowerVR SGX544MP2 - Medfield   -
+ * PowerVR SGX545    - Cedartrail - Intel GMA 3600, Intel Atom D2500, N2600
+ * PowerVR SGX545    - Cedartrail - Intel GMA 3650, Intel Atom D2550, D2700,
+ *                                  N2800
+ */
 static DEFINE_PCI_DEVICE_TABLE(pciidlist) = {
         { 0x8086, 0x8108, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &psb_chip_ops },
         { 0x8086, 0x8109, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &psb_chip_ops },
 #if defined(CONFIG_DRM_GMA600)
-        { 0x8086, 0x4100, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops},
-        { 0x8086, 0x4101, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops},
-        { 0x8086, 0x4102, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops},
-        { 0x8086, 0x4103, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops},
-        { 0x8086, 0x4104, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops},
-        { 0x8086, 0x4105, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops},
-        { 0x8086, 0x4106, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops},
-        { 0x8086, 0x4107, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops},
-        /* Atom E620 */
-        { 0x8086, 0x4108, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops},
+        { 0x8086, 0x4100, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
+        { 0x8086, 0x4101, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
+        { 0x8086, 0x4102, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
+        { 0x8086, 0x4103, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
+        { 0x8086, 0x4104, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
+        { 0x8086, 0x4105, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
+        { 0x8086, 0x4106, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
+        { 0x8086, 0x4107, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
+        { 0x8086, 0x4108, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &oaktrail_chip_ops },
 #endif
 #if defined(CONFIG_DRM_MEDFIELD)
-        {0x8086, 0x0130, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops},
-        {0x8086, 0x0131, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops},
-        {0x8086, 0x0132, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops},
-        {0x8086, 0x0133, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops},
-        {0x8086, 0x0134, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops},
-        {0x8086, 0x0135, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops},
-        {0x8086, 0x0136, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops},
-        {0x8086, 0x0137, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops},
+        { 0x8086, 0x0130, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops },
+        { 0x8086, 0x0131, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops },
+        { 0x8086, 0x0132, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops },
+        { 0x8086, 0x0133, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops },
+        { 0x8086, 0x0134, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops },
+        { 0x8086, 0x0135, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops },
+        { 0x8086, 0x0136, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops },
+        { 0x8086, 0x0137, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &mdfld_chip_ops },
 #endif
 #if defined(CONFIG_DRM_GMA3600)
-        { 0x8086, 0x0be0, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0be1, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0be2, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0be3, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0be4, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0be5, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0be6, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0be7, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0be8, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0be9, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0bea, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0beb, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0bec, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0bed, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0bee, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
-        { 0x8086, 0x0bef, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops},
+        { 0x8086, 0x0be0, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0be1, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0be2, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0be3, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0be4, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0be5, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0be6, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0be7, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0be8, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0be9, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0bea, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0beb, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0bec, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0bed, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0bee, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
+        { 0x8086, 0x0bef, PCI_ANY_ID, PCI_ANY_ID, 0, 0, (long) &cdv_chip_ops },
 #endif
         { 0, }
 };
@@ -95,59 +103,10 @@ MODULE_DEVICE_TABLE(pci, pciidlist);
 /*
  * Standard IOCTLs.
  */
-
-#define DRM_IOCTL_GMA_ADB       \
-                DRM_IOWR(DRM_GMA_ADB + DRM_COMMAND_BASE, uint32_t)
-#define DRM_IOCTL_GMA_MODE_OPERATION    \
-                DRM_IOWR(DRM_GMA_MODE_OPERATION + DRM_COMMAND_BASE, \
-                         struct drm_psb_mode_operation_arg)
-#define DRM_IOCTL_GMA_STOLEN_MEMORY     \
-                DRM_IOWR(DRM_GMA_STOLEN_MEMORY + DRM_COMMAND_BASE, \
-                         struct drm_psb_stolen_memory_arg)
-#define DRM_IOCTL_GMA_GAMMA     \
-                DRM_IOWR(DRM_GMA_GAMMA + DRM_COMMAND_BASE, \
-                         struct drm_psb_dpst_lut_arg)
-#define DRM_IOCTL_GMA_DPST_BL   \
-                DRM_IOWR(DRM_GMA_DPST_BL + DRM_COMMAND_BASE, \
-                         uint32_t)
-#define DRM_IOCTL_GMA_GET_PIPE_FROM_CRTC_ID     \
-                DRM_IOWR(DRM_GMA_GET_PIPE_FROM_CRTC_ID + DRM_COMMAND_BASE, \
-                         struct drm_psb_get_pipe_from_crtc_id_arg)
-#define DRM_IOCTL_GMA_GEM_CREATE        \
-                DRM_IOWR(DRM_GMA_GEM_CREATE + DRM_COMMAND_BASE, \
-                         struct drm_psb_gem_create)
-#define DRM_IOCTL_GMA_GEM_MMAP  \
-                DRM_IOWR(DRM_GMA_GEM_MMAP + DRM_COMMAND_BASE, \
-                         struct drm_psb_gem_mmap)
-
-static int psb_adb_ioctl(struct drm_device *dev, void *data,
-                         struct drm_file *file_priv);
-static int psb_mode_operation_ioctl(struct drm_device *dev, void *data,
-                                    struct drm_file *file_priv);
-static int psb_stolen_memory_ioctl(struct drm_device *dev, void *data,
-                                   struct drm_file *file_priv);
-static int psb_gamma_ioctl(struct drm_device *dev, void *data,
-                           struct drm_file *file_priv);
-static int psb_dpst_bl_ioctl(struct drm_device *dev, void *data,
-                             struct drm_file *file_priv);
-
 static const struct drm_ioctl_desc psb_ioctls[] = {
-        DRM_IOCTL_DEF_DRV(GMA_ADB, psb_adb_ioctl, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(GMA_MODE_OPERATION, psb_mode_operation_ioctl,
-                      DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(GMA_STOLEN_MEMORY, psb_stolen_memory_ioctl,
-                      DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(GMA_GAMMA, psb_gamma_ioctl, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(GMA_DPST_BL, psb_dpst_bl_ioctl, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(GMA_GET_PIPE_FROM_CRTC_ID,
-                                        psb_intel_get_pipe_from_crtc_id, 0),
-        DRM_IOCTL_DEF_DRV(GMA_GEM_CREATE, psb_gem_create_ioctl,
-                                                DRM_UNLOCKED | DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(GMA_GEM_MMAP, psb_gem_mmap_ioctl,
-                                                DRM_UNLOCKED | DRM_AUTH),
 };
 
-static void psb_lastclose(struct drm_device *dev)
+static void psb_driver_lastclose(struct drm_device *dev)
 {
         int ret;
         struct drm_psb_private *dev_priv = dev->dev_private;
@@ -169,19 +128,14 @@ static int psb_do_init(struct drm_device *dev)
 
         uint32_t stolen_gtt;
 
-        int ret = -ENOMEM;
-
         if (pg->mmu_gatt_start & 0x0FFFFFFF) {
                 dev_err(dev->dev, "Gatt must be 256M aligned. This is a bug.\n");
-                ret = -EINVAL;
-                goto out_err;
+                return -EINVAL;
         }
 
-
         stolen_gtt = (pg->stolen_size >> PAGE_SHIFT) * 4;
         stolen_gtt = (stolen_gtt + PAGE_SIZE - 1) >> PAGE_SHIFT;
-        stolen_gtt =
-            (stolen_gtt < pg->gtt_pages) ? stolen_gtt : pg->gtt_pages;
+        stolen_gtt = (stolen_gtt < pg->gtt_pages) ? stolen_gtt : pg->gtt_pages;
 
         dev_priv->gatt_free_offset = pg->mmu_gatt_start +
             (stolen_gtt << PAGE_SHIFT) * 1024;
@@ -192,23 +146,26 @@ static int psb_do_init(struct drm_device *dev)
         PSB_WSGX32(0x00000000, PSB_CR_BIF_BANK0);
         PSB_WSGX32(0x00000000, PSB_CR_BIF_BANK1);
         PSB_RSGX32(PSB_CR_BIF_BANK1);
-        PSB_WSGX32(PSB_RSGX32(PSB_CR_BIF_CTRL) | _PSB_MMU_ER_MASK,
-                                                        PSB_CR_BIF_CTRL);
+
+        /* Do not bypass any MMU access, let them pagefault instead */
+        PSB_WSGX32((PSB_RSGX32(PSB_CR_BIF_CTRL) & ~_PSB_MMU_ER_MASK),
+                   PSB_CR_BIF_CTRL);
+        PSB_RSGX32(PSB_CR_BIF_CTRL);
+
         psb_spank(dev_priv);
 
         /* mmu_gatt ?? */
         PSB_WSGX32(pg->gatt_start, PSB_CR_BIF_TWOD_REQ_BASE);
+        PSB_RSGX32(PSB_CR_BIF_TWOD_REQ_BASE); /* Post */
+
         return 0;
-out_err:
-        return ret;
 }
 
 static int psb_driver_unload(struct drm_device *dev)
 {
         struct drm_psb_private *dev_priv = dev->dev_private;
 
-        /* Kill vblank etc here */
-
+        /* TODO: Kill vblank etc here */
 
         if (dev_priv) {
                 if (dev_priv->backlight_device)
@@ -268,8 +225,7 @@ static int psb_driver_unload(struct drm_device *dev)
         return 0;
 }
 
-
-static int psb_driver_load(struct drm_device *dev, unsigned long chipset)
+static int psb_driver_load(struct drm_device *dev, unsigned long flags)
 {
         struct drm_psb_private *dev_priv;
         unsigned long resource_start, resource_len;
@@ -277,15 +233,19 @@ static int psb_driver_load(struct drm_device *dev, unsigned long chipset)
         int ret = -ENOMEM;
         struct drm_connector *connector;
         struct gma_encoder *gma_encoder;
+        struct psb_gtt *pg;
 
+        /* allocating and initializing driver private data */
         dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL);
         if (dev_priv == NULL)
                 return -ENOMEM;
 
-        dev_priv->ops = (struct psb_ops *)chipset;
+        dev_priv->ops = (struct psb_ops *)flags;
         dev_priv->dev = dev;
         dev->dev_private = (void *) dev_priv;
 
+        pg = &dev_priv->gtt;
+
         pci_set_master(dev->pdev);
 
         dev_priv->num_pipe = dev_priv->ops->pipes;
@@ -347,9 +307,7 @@ static int psb_driver_load(struct drm_device *dev, unsigned long chipset)
         if (ret)
                 goto out_err;
 
-        dev_priv->mmu = psb_mmu_driver_init((void *)0,
-                                        drm_psb_trap_pagefaults, 0,
-                                        dev_priv);
+        dev_priv->mmu = psb_mmu_driver_init(dev, 1, 0, 0);
         if (!dev_priv->mmu)
                 goto out_err;
 
@@ -357,18 +315,27 @@ static int psb_driver_load(struct drm_device *dev, unsigned long chipset)
         if (!dev_priv->pf_pd)
                 goto out_err;
 
-        psb_mmu_set_pd_context(psb_mmu_get_default_pd(dev_priv->mmu), 0);
-        psb_mmu_set_pd_context(dev_priv->pf_pd, 1);
-
         ret = psb_do_init(dev);
         if (ret)
                 return ret;
 
+        /* Add stolen memory to SGX MMU */
+        down_read(&pg->sem);
+        ret = psb_mmu_insert_pfn_sequence(psb_mmu_get_default_pd(dev_priv->mmu),
+                                          dev_priv->stolen_base >> PAGE_SHIFT,
+                                          pg->gatt_start,
+                                          pg->stolen_size >> PAGE_SHIFT, 0);
+        up_read(&pg->sem);
+
+        psb_mmu_set_pd_context(psb_mmu_get_default_pd(dev_priv->mmu), 0);
+        psb_mmu_set_pd_context(dev_priv->pf_pd, 1);
+
         PSB_WSGX32(0x20000000, PSB_CR_PDS_EXEC_BASE);
         PSB_WSGX32(0x30000000, PSB_CR_BIF_3D_REQ_BASE);
 
         acpi_video_register();
 
+        /* Setup vertical blanking handling */
         ret = drm_vblank_init(dev, dev_priv->num_pipe);
         if (ret)
                 goto out_err;
@@ -390,9 +357,7 @@ static int psb_driver_load(struct drm_device *dev, unsigned long chipset)
         drm_irq_install(dev);
 
         dev->vblank_disable_allowed = true;
-
         dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
-
         dev->driver->get_vblank_counter = psb_get_vblank_counter;
 
         psb_modeset_init(dev);
@@ -416,11 +381,11 @@ static int psb_driver_load(struct drm_device *dev, unsigned long chipset)
                 return ret;
         psb_intel_opregion_enable_asle(dev);
 #if 0
-        /*enable runtime pm at last*/
+        /* Enable runtime pm at last */
         pm_runtime_enable(&dev->pdev->dev);
         pm_runtime_set_active(&dev->pdev->dev);
 #endif
-        /*Intel drm driver load is done, continue doing pvr load*/
+        /* Intel drm driver load is done, continue doing pvr load */
         return 0;
 out_err:
         psb_driver_unload(dev);
@@ -442,161 +407,6 @@ static inline void get_brightness(struct backlight_device *bd)
 #endif
 }
 
-static int psb_dpst_bl_ioctl(struct drm_device *dev, void *data,
-                       struct drm_file *file_priv)
-{
-        struct drm_psb_private *dev_priv = psb_priv(dev);
-        uint32_t *arg = data;
-
-        dev_priv->blc_adj2 = *arg;
-        get_brightness(dev_priv->backlight_device);
-        return 0;
-}
-
-static int psb_adb_ioctl(struct drm_device *dev, void *data,
-                        struct drm_file *file_priv)
-{
-        struct drm_psb_private *dev_priv = psb_priv(dev);
-        uint32_t *arg = data;
-
-        dev_priv->blc_adj1 = *arg;
-        get_brightness(dev_priv->backlight_device);
-        return 0;
-}
-
-static int psb_gamma_ioctl(struct drm_device *dev, void *data,
-                           struct drm_file *file_priv)
-{
-        struct drm_psb_dpst_lut_arg *lut_arg = data;
-        struct drm_mode_object *obj;
-        struct drm_crtc *crtc;
-        struct drm_connector *connector;
-        struct gma_crtc *gma_crtc;
-        int i = 0;
-        int32_t obj_id;
-
-        obj_id = lut_arg->output_id;
-        obj = drm_mode_object_find(dev, obj_id, DRM_MODE_OBJECT_CONNECTOR);
-        if (!obj) {
-                dev_dbg(dev->dev, "Invalid Connector object.\n");
-                return -ENOENT;
-        }
-
-        connector = obj_to_connector(obj);
-        crtc = connector->encoder->crtc;
-        gma_crtc = to_gma_crtc(crtc);
-
-        for (i = 0; i < 256; i++)
-                gma_crtc->lut_adj[i] = lut_arg->lut[i];
-
-        gma_crtc_load_lut(crtc);
-
-        return 0;
-}
-
-static int psb_mode_operation_ioctl(struct drm_device *dev, void *data,
-                                struct drm_file *file_priv)
-{
-        uint32_t obj_id;
-        uint16_t op;
-        struct drm_mode_modeinfo *umode;
-        struct drm_display_mode *mode = NULL;
-        struct drm_psb_mode_operation_arg *arg;
-        struct drm_mode_object *obj;
-        struct drm_connector *connector;
-        struct drm_connector_helper_funcs *connector_funcs;
-        int ret = 0;
-        int resp = MODE_OK;
-
-        arg = (struct drm_psb_mode_operation_arg *)data;
-        obj_id = arg->obj_id;
-        op = arg->operation;
-
-        switch (op) {
-        case PSB_MODE_OPERATION_MODE_VALID:
-                umode = &arg->mode;
-
-                drm_modeset_lock_all(dev);
-
-                obj = drm_mode_object_find(dev, obj_id,
-                                        DRM_MODE_OBJECT_CONNECTOR);
-                if (!obj) {
-                        ret = -ENOENT;
-                        goto mode_op_out;
-                }
-
-                connector = obj_to_connector(obj);
-
-                mode = drm_mode_create(dev);
-                if (!mode) {
-                        ret = -ENOMEM;
-                        goto mode_op_out;
-                }
-
-                /* drm_crtc_convert_umode(mode, umode); */
-                {
-                        mode->clock = umode->clock;
-                        mode->hdisplay = umode->hdisplay;
-                        mode->hsync_start = umode->hsync_start;
-                        mode->hsync_end = umode->hsync_end;
-                        mode->htotal = umode->htotal;
-                        mode->hskew = umode->hskew;
-                        mode->vdisplay = umode->vdisplay;
-                        mode->vsync_start = umode->vsync_start;
-                        mode->vsync_end = umode->vsync_end;
-                        mode->vtotal = umode->vtotal;
-                        mode->vscan = umode->vscan;
-                        mode->vrefresh = umode->vrefresh;
-                        mode->flags = umode->flags;
-                        mode->type = umode->type;
-                        strncpy(mode->name, umode->name, DRM_DISPLAY_MODE_LEN);
-                        mode->name[DRM_DISPLAY_MODE_LEN-1] = 0;
-                }
-
-                connector_funcs = (struct drm_connector_helper_funcs *)
-                                   connector->helper_private;
-
-                if (connector_funcs->mode_valid) {
-                        resp = connector_funcs->mode_valid(connector, mode);
-                        arg->data = resp;
-                }
-
-                /*do some clean up work*/
-                if (mode)
-                        drm_mode_destroy(dev, mode);
-mode_op_out:
-                drm_modeset_unlock_all(dev);
-                return ret;
-
-        default:
-                dev_dbg(dev->dev, "Unsupported psb mode operation\n");
-                return -EOPNOTSUPP;
-        }
-
-        return 0;
-}
-
-static int psb_stolen_memory_ioctl(struct drm_device *dev, void *data,
-                                   struct drm_file *file_priv)
-{
-        struct drm_psb_private *dev_priv = psb_priv(dev);
-        struct drm_psb_stolen_memory_arg *arg = data;
-
-        arg->base = dev_priv->stolen_base;
-        arg->size = dev_priv->vram_stolen_size;
-
-        return 0;
-}
-
-static int psb_driver_open(struct drm_device *dev, struct drm_file *priv)
-{
-        return 0;
-}
-
-static void psb_driver_close(struct drm_device *dev, struct drm_file *priv)
-{
-}
-
 static long psb_unlocked_ioctl(struct file *filp, unsigned int cmd,
                                unsigned long arg)
 {
@@ -614,15 +424,21 @@ static long psb_unlocked_ioctl(struct file *filp, unsigned int cmd,
         /* FIXME: do we need to wrap the other side of this */
 }
 
-
-/* When a client dies:
+/*
+ * When a client dies:
  *    - Check for and clean up flipped page state
  */
 static void psb_driver_preclose(struct drm_device *dev, struct drm_file *priv)
 {
 }
 
-static void psb_remove(struct pci_dev *pdev)
+static int psb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+        return drm_get_pci_dev(pdev, ent, &driver);
+}
+
+
+static void psb_pci_remove(struct pci_dev *pdev)
 {
         struct drm_device *dev = pci_get_drvdata(pdev);
         drm_put_dev(dev);
@@ -657,11 +473,12 @@ static const struct file_operations psb_gem_fops = {
 
 static struct drm_driver driver = {
         .driver_features = DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | \
-                           DRIVER_MODESET | DRIVER_GEM ,
+                           DRIVER_MODESET | DRIVER_GEM,
         .load = psb_driver_load,
         .unload = psb_driver_unload,
+        .lastclose = psb_driver_lastclose,
+        .preclose = psb_driver_preclose,
 
-        .ioctls = psb_ioctls,
         .num_ioctls = DRM_ARRAY_SIZE(psb_ioctls),
         .device_is_agp = psb_driver_device_is_agp,
         .irq_preinstall = psb_irq_preinstall,
@@ -671,40 +488,31 @@ static struct drm_driver driver = {
         .enable_vblank = psb_enable_vblank,
         .disable_vblank = psb_disable_vblank,
         .get_vblank_counter = psb_get_vblank_counter,
-        .lastclose = psb_lastclose,
-        .open = psb_driver_open,
-        .preclose = psb_driver_preclose,
-        .postclose = psb_driver_close,
 
         .gem_free_object = psb_gem_free_object,
         .gem_vm_ops = &psb_gem_vm_ops,
+
         .dumb_create = psb_gem_dumb_create,
         .dumb_map_offset = psb_gem_dumb_map_gtt,
         .dumb_destroy = drm_gem_dumb_destroy,
+        .ioctls = psb_ioctls,
         .fops = &psb_gem_fops,
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
-        .date = PSB_DRM_DRIVER_DATE,
-        .major = PSB_DRM_DRIVER_MAJOR,
-        .minor = PSB_DRM_DRIVER_MINOR,
-        .patchlevel = PSB_DRM_DRIVER_PATCHLEVEL
+        .date = DRIVER_DATE,
+        .major = DRIVER_MAJOR,
+        .minor = DRIVER_MINOR,
+        .patchlevel = DRIVER_PATCHLEVEL
 };
 
 static struct pci_driver psb_pci_driver = {
         .name = DRIVER_NAME,
         .id_table = pciidlist,
-        .probe = psb_probe,
-        .remove = psb_remove,
-        .driver = {
-                .pm = &psb_pm_ops,
-        }
+        .probe = psb_pci_probe,
+        .remove = psb_pci_remove,
+        .driver.pm = &psb_pm_ops,
 };
 
-static int psb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
-{
-        return drm_get_pci_dev(pdev, ent, &driver);
-}
-
 static int __init psb_init(void)
 {
         return drm_pci_init(&driver, &psb_pci_driver);
@@ -718,6 +526,6 @@ static void __exit psb_exit(void)
 late_initcall(psb_init);
 module_exit(psb_exit);
 
-MODULE_AUTHOR("Alan Cox <alan@linux.intel.com> and others");
+MODULE_AUTHOR(DRIVER_AUTHOR);
 MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE("GPL");
+MODULE_LICENSE(DRIVER_LICENSE);
diff --git a/drivers/gpu/drm/gma500/psb_drv.h b/drivers/gpu/drm/gma500/psb_drv.h
index 5ad6a03e477e..55ebe2bd88dd 100644
--- a/drivers/gpu/drm/gma500/psb_drv.h
+++ b/drivers/gpu/drm/gma500/psb_drv.h
@@ -33,6 +33,18 @@
 #include "power.h"
 #include "opregion.h"
 #include "oaktrail.h"
+#include "mmu.h"
+
+#define DRIVER_AUTHOR "Alan Cox <alan@linux.intel.com> and others"
+#define DRIVER_LICENSE "GPL"
+
+#define DRIVER_NAME "gma500"
+#define DRIVER_DESC "DRM driver for the Intel GMA500, GMA600, GMA3600, GMA3650"
+#define DRIVER_DATE "20140314"
+
+#define DRIVER_MAJOR 1
+#define DRIVER_MINOR 0
+#define DRIVER_PATCHLEVEL 0
 
 /* Append new drm mode definition here, align with libdrm definition */
 #define DRM_MODE_SCALE_NO_SCALE         2
@@ -49,21 +61,7 @@ enum {
 #define IS_MFLD(dev) (((dev)->pdev->device & 0xfff8) == 0x0130)
 #define IS_CDV(dev) (((dev)->pdev->device & 0xfff0) == 0x0be0)
 
-/*
- * Driver definitions
- */
-
-#define DRIVER_NAME "gma500"
-#define DRIVER_DESC "DRM driver for the Intel GMA500"
-
-#define PSB_DRM_DRIVER_DATE "2011-06-06"
-#define PSB_DRM_DRIVER_MAJOR 1
-#define PSB_DRM_DRIVER_MINOR 0
-#define PSB_DRM_DRIVER_PATCHLEVEL 0
-
-/*
- *      Hardware offsets
- */
+/* Hardware offsets */
 #define PSB_VDC_OFFSET           0x00000000
 #define PSB_VDC_SIZE             0x000080000
 #define MRST_MMIO_SIZE           0x0000C0000
@@ -71,16 +69,14 @@ enum {
 #define PSB_SGX_SIZE             0x8000
 #define PSB_SGX_OFFSET           0x00040000
 #define MRST_SGX_OFFSET          0x00080000
-/*
- *      PCI resource identifiers
- */
+
+/* PCI resource identifiers */
 #define PSB_MMIO_RESOURCE        0
 #define PSB_AUX_RESOURCE         0
 #define PSB_GATT_RESOURCE        2
 #define PSB_GTT_RESOURCE         3
-/*
- *      PCI configuration
- */
+
+/* PCI configuration */
 #define PSB_GMCH_CTRL            0x52
 #define PSB_BSM                  0x5C
 #define _PSB_GMCH_ENABLED        0x4
@@ -88,37 +84,29 @@ enum {
 #define _PSB_PGETBL_ENABLED      0x00000001
 #define PSB_SGX_2D_SLAVE_PORT    0x4000
 
-/* To get rid of */
+/* TODO: To get rid of */
 #define PSB_TT_PRIV0_LIMIT       (256*1024*1024)
 #define PSB_TT_PRIV0_PLIMIT      (PSB_TT_PRIV0_LIMIT >> PAGE_SHIFT)
 
-/*
- *      SGX side MMU definitions (these can probably go)
- */
+/* SGX side MMU definitions (these can probably go) */
 
-/*
- *      Flags for external memory type field.
- */
+/* Flags for external memory type field */
 #define PSB_MMU_CACHED_MEMORY     0x0001        /* Bind to MMU only */
 #define PSB_MMU_RO_MEMORY         0x0002        /* MMU RO memory */
 #define PSB_MMU_WO_MEMORY         0x0004        /* MMU WO memory */
-/*
- *      PTE's and PDE's
- */
+
+/* PTE's and PDE's */
 #define PSB_PDE_MASK              0x003FFFFF
 #define PSB_PDE_SHIFT             22
 #define PSB_PTE_SHIFT             12
-/*
- *      Cache control
- */
+
+/* Cache control */
 #define PSB_PTE_VALID             0x0001        /* PTE / PDE valid */
 #define PSB_PTE_WO                0x0002        /* Write only */
 #define PSB_PTE_RO                0x0004        /* Read only */
 #define PSB_PTE_CACHED            0x0008        /* CPU cache coherent */
 
-/*
- *      VDC registers and bits
- */
+/* VDC registers and bits */
 #define PSB_MSVDX_CLOCKGATING     0x2064
 #define PSB_TOPAZ_CLOCKGATING     0x2068
 #define PSB_HWSTAM                0x2098
@@ -265,6 +253,7 @@ struct psb_intel_opregion {
         struct opregion_asle *asle;
         void *vbt;
         u32 __iomem *lid_state;
+        struct work_struct asle_work;
 };
 
 struct sdvo_device_mapping {
@@ -283,10 +272,7 @@ struct intel_gmbus {
         u32 reg0;
 };
 
-/*
- *      Register offset maps
- */
-
+/* Register offset maps */
 struct psb_offset {
         u32     fp0;
         u32     fp1;
@@ -320,9 +306,7 @@ struct psb_offset {
  *      update the register cache instead.
  */
 
-/*
- *      Common status for pipes.
- */
+/* Common status for pipes */
 struct psb_pipe {
         u32     fp0;
         u32     fp1;
@@ -482,35 +466,24 @@ struct drm_psb_private {
         struct psb_mmu_driver *mmu;
         struct psb_mmu_pd *pf_pd;
 
-        /*
-         * Register base
-         */
-
+        /* Register base */
         uint8_t __iomem *sgx_reg;
         uint8_t __iomem *vdc_reg;
         uint8_t __iomem *aux_reg; /* Auxillary vdc pipe regs */
         uint32_t gatt_free_offset;
 
-        /*
-         * Fencing / irq.
-         */
-
+        /* Fencing / irq */
         uint32_t vdc_irq_mask;
         uint32_t pipestat[PSB_NUM_PIPE];
 
         spinlock_t irqmask_lock;
 
-        /*
-         * Power
-         */
-
+        /* Power */
         bool suspended;
         bool display_power;
         int display_count;
 
-        /*
-         * Modesetting
-         */
+        /* Modesetting */
         struct psb_intel_mode_device mode_dev;
         bool modeset;   /* true if we have done the mode_device setup */
 
@@ -518,15 +491,10 @@ struct drm_psb_private {
         struct drm_crtc *pipe_to_crtc_mapping[PSB_NUM_PIPE];
         uint32_t num_pipe;
 
-        /*
-         * OSPM info (Power management base) (can go ?)
-         */
+        /* OSPM info (Power management base) (TODO: can go ?) */
         uint32_t ospm_base;
 
-        /*
-         * Sizes info
-         */
-
+        /* Sizes info */
         u32 fuse_reg_value;
         u32 video_device_fuse;
 
@@ -546,9 +514,7 @@ struct drm_psb_private {
         struct drm_property *broadcast_rgb_property;
         struct drm_property *force_audio_property;
 
-        /*
-         * LVDS info
-         */
+        /* LVDS info */
         int backlight_duty_cycle;       /* restore backlight to this value */
         bool panel_wants_dither;
         struct drm_display_mode *panel_fixed_mode;
@@ -582,34 +548,23 @@ struct drm_psb_private {
         /* Oaktrail HDMI state */
         struct oaktrail_hdmi_dev *hdmi_priv;
         
-        /*
-         * Register state
-         */
-
+        /* Register state */
         struct psb_save_area regs;
 
         /* MSI reg save */
         uint32_t msi_addr;
         uint32_t msi_data;
 
-        /*
-         * Hotplug handling
-         */
-
+        /* Hotplug handling */
         struct work_struct hotplug_work;
 
-        /*
-         * LID-Switch
-         */
+        /* LID-Switch */
         spinlock_t lid_lock;
         struct timer_list lid_timer;
         struct psb_intel_opregion opregion;
         u32 lid_last_state;
 
-        /*
-         * Watchdog
-         */
-
+        /* Watchdog */
         uint32_t apm_reg;
         uint16_t apm_base;
 
@@ -629,9 +584,7 @@ struct drm_psb_private {
         /* 2D acceleration */
         spinlock_t lock_2d;
 
-        /*
-         * Panel brightness
-         */
+        /* Panel brightness */
         int brightness;
         int brightness_adjusted;
 
@@ -664,10 +617,7 @@ struct drm_psb_private {
 };
 
 
-/*
- *      Operations for each board type
- */
- 
+/* Operations for each board type */
 struct psb_ops {
         const char *name;
         unsigned int accel_2d:1;
@@ -713,8 +663,6 @@ struct psb_ops {
 
 
 
-struct psb_mmu_driver;
-
 extern int drm_crtc_probe_output_modes(struct drm_device *dev, int, int);
 extern int drm_pick_crtcs(struct drm_device *dev);
 
@@ -723,52 +671,7 @@ static inline struct drm_psb_private *psb_priv(struct drm_device *dev)
         return (struct drm_psb_private *) dev->dev_private;
 }
 
-/*
- * MMU stuff.
- */
-
-extern struct psb_mmu_driver *psb_mmu_driver_init(uint8_t __iomem * registers,
-                                        int trap_pagefaults,
-                                        int invalid_type,
-                                        struct drm_psb_private *dev_priv);
-extern void psb_mmu_driver_takedown(struct psb_mmu_driver *driver);
-extern struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver
-                                                 *driver);
-extern void psb_mmu_mirror_gtt(struct psb_mmu_pd *pd, uint32_t mmu_offset,
-                               uint32_t gtt_start, uint32_t gtt_pages);
-extern struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
-                                           int trap_pagefaults,
-                                           int invalid_type);
-extern void psb_mmu_free_pagedir(struct psb_mmu_pd *pd);
-extern void psb_mmu_flush(struct psb_mmu_driver *driver, int rc_prot);
-extern void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
-                                        unsigned long address,
-                                        uint32_t num_pages);
-extern int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd,
-                                       uint32_t start_pfn,
-                                       unsigned long address,
-                                       uint32_t num_pages, int type);
-extern int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,
-                                  unsigned long *pfn);
-
-/*
- * Enable / disable MMU for different requestors.
- */
-
-
-extern void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context);
-extern int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
-                                unsigned long address, uint32_t num_pages,
-                                uint32_t desired_tile_stride,
-                                uint32_t hw_tile_stride, int type);
-extern void psb_mmu_remove_pages(struct psb_mmu_pd *pd,
-                                 unsigned long address, uint32_t num_pages,
-                                 uint32_t desired_tile_stride,
-                                 uint32_t hw_tile_stride);
-/*
- *psb_irq.c
- */
-
+/* psb_irq.c */
 extern irqreturn_t psb_irq_handler(int irq, void *arg);
 extern int psb_irq_enable_dpst(struct drm_device *dev);
 extern int psb_irq_disable_dpst(struct drm_device *dev);
@@ -791,24 +694,17 @@ psb_disable_pipestat(struct drm_psb_private *dev_priv, int pipe, u32 mask);
 
 extern u32 psb_get_vblank_counter(struct drm_device *dev, int crtc);
 
-/*
- * framebuffer.c
- */
+/* framebuffer.c */
 extern int psbfb_probed(struct drm_device *dev);
 extern int psbfb_remove(struct drm_device *dev,
                         struct drm_framebuffer *fb);
-/*
- * accel_2d.c
- */
+/* accel_2d.c */
 extern void psbfb_copyarea(struct fb_info *info,
                                         const struct fb_copyarea *region);
 extern int psbfb_sync(struct fb_info *info);
 extern void psb_spank(struct drm_psb_private *dev_priv);
 
-/*
- * psb_reset.c
- */
-
+/* psb_reset.c */
 extern void psb_lid_timer_init(struct drm_psb_private *dev_priv);
 extern void psb_lid_timer_takedown(struct drm_psb_private *dev_priv);
 extern void psb_print_pagefault(struct drm_psb_private *dev_priv);
@@ -867,9 +763,7 @@ extern const struct psb_ops mdfld_chip_ops;
 /* cdv_device.c */
 extern const struct psb_ops cdv_chip_ops;
 
-/*
- * Debug print bits setting
- */
+/* Debug print bits setting */
 #define PSB_D_GENERAL (1 << 0)
 #define PSB_D_INIT    (1 << 1)
 #define PSB_D_IRQ     (1 << 2)
@@ -885,10 +779,7 @@ extern const struct psb_ops cdv_chip_ops;
 
 extern int drm_idle_check_interval;
 
-/*
- *      Utilities
- */
-
+/* Utilities */
 static inline u32 MRST_MSG_READ32(uint port, uint offset)
 {
         int mcr = (0xD0<<24) | (port << 16) | (offset << 8);
diff --git a/drivers/gpu/drm/gma500/psb_intel_display.c b/drivers/gpu/drm/gma500/psb_intel_display.c
index c8841ac6c8f1..87b50ba64ed4 100644
--- a/drivers/gpu/drm/gma500/psb_intel_display.c
+++ b/drivers/gpu/drm/gma500/psb_intel_display.c
@@ -120,7 +120,7 @@ static int psb_intel_crtc_mode_set(struct drm_crtc *crtc,
         const struct gma_limit_t *limit;
 
         /* No scan out no play */
-        if (crtc->fb == NULL) {
+        if (crtc->primary->fb == NULL) {
                 crtc_funcs->mode_set_base(crtc, x, y, old_fb);
                 return 0;
         }
@@ -469,7 +469,8 @@ static void psb_intel_cursor_init(struct drm_device *dev,
                 /* Allocate 4 pages of stolen mem for a hardware cursor. That
                  * is enough for the 64 x 64 ARGB cursors we support.
                  */
-                cursor_gt = psb_gtt_alloc_range(dev, 4 * PAGE_SIZE, "cursor", 1);
+                cursor_gt = psb_gtt_alloc_range(dev, 4 * PAGE_SIZE, "cursor", 1,
+                                                PAGE_SIZE);
                 if (!cursor_gt) {
                         gma_crtc->cursor_gt = NULL;
                         goto out;
@@ -554,33 +555,6 @@ void psb_intel_crtc_init(struct drm_device *dev, int pipe,
         gma_crtc->active = true;
 }
 
-int psb_intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
-                                struct drm_file *file_priv)
-{
-        struct drm_psb_private *dev_priv = dev->dev_private;
-        struct drm_psb_get_pipe_from_crtc_id_arg *pipe_from_crtc_id = data;
-        struct drm_mode_object *drmmode_obj;
-        struct gma_crtc *crtc;
-
-        if (!dev_priv) {
-                dev_err(dev->dev, "called with no initialization\n");
-                return -EINVAL;
-        }
-
-        drmmode_obj = drm_mode_object_find(dev, pipe_from_crtc_id->crtc_id,
-                        DRM_MODE_OBJECT_CRTC);
-
-        if (!drmmode_obj) {
-                dev_err(dev->dev, "no such CRTC id\n");
-                return -ENOENT;
-        }
-
-        crtc = to_gma_crtc(obj_to_crtc(drmmode_obj));
-        pipe_from_crtc_id->pipe = crtc->pipe;
-
-        return 0;
-}
-
 struct drm_crtc *psb_intel_get_crtc_from_pipe(struct drm_device *dev, int pipe)
 {
         struct drm_crtc *crtc = NULL;
diff --git a/drivers/gpu/drm/gma500/psb_intel_drv.h b/drivers/gpu/drm/gma500/psb_intel_drv.h
index dc2c8eb030fa..336bd3aa1a06 100644
--- a/drivers/gpu/drm/gma500/psb_intel_drv.h
+++ b/drivers/gpu/drm/gma500/psb_intel_drv.h
@@ -238,8 +238,6 @@ static inline struct gma_encoder *gma_attached_encoder(
 
 extern struct drm_display_mode *psb_intel_crtc_mode_get(struct drm_device *dev,
                                                     struct drm_crtc *crtc);
-extern int psb_intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
-                                struct drm_file *file_priv);
 extern struct drm_crtc *psb_intel_get_crtc_from_pipe(struct drm_device *dev,
                                                  int pipe);
 extern struct drm_connector *psb_intel_sdvo_find(struct drm_device *dev,
diff --git a/drivers/gpu/drm/gma500/psb_intel_lvds.c b/drivers/gpu/drm/gma500/psb_intel_lvds.c
index 32342f6990d9..d7778d0472c1 100644
--- a/drivers/gpu/drm/gma500/psb_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/psb_intel_lvds.c
@@ -614,7 +614,7 @@ int psb_intel_lvds_set_property(struct drm_connector *connector,
                                                       &crtc->saved_mode,
                                                       encoder->crtc->x,
                                                       encoder->crtc->y,
-                                                      encoder->crtc->fb))
+                                                      encoder->crtc->primary->fb))
                                 goto set_prop_error;
                 }
         } else if (!strcmp(property->name, "backlight")) {
@@ -777,6 +777,7 @@ void psb_intel_lvds_init(struct drm_device *dev,
          * Attempt to get the fixed panel mode from DDC.  Assume that the
          * preferred mode is the right one.
          */
+        mutex_lock(&dev->mode_config.mutex);
         psb_intel_ddc_get_modes(connector, &lvds_priv->ddc_bus->adapter);
         list_for_each_entry(scan, &connector->probed_modes, head) {
                 if (scan->type & DRM_MODE_TYPE_PREFERRED) {
@@ -827,10 +828,12 @@ void psb_intel_lvds_init(struct drm_device *dev,
          * actually having one.
          */
 out:
+        mutex_unlock(&dev->mode_config.mutex);
         drm_sysfs_connector_add(connector);
         return;
 
 failed_find:
+        mutex_unlock(&dev->mode_config.mutex);
         if (lvds_priv->ddc_bus)
                 psb_intel_i2c_destroy(lvds_priv->ddc_bus);
 failed_ddc:
diff --git a/drivers/gpu/drm/gma500/psb_intel_sdvo.c b/drivers/gpu/drm/gma500/psb_intel_sdvo.c
index 07d3a9e6d79b..deeb0829b129 100644
--- a/drivers/gpu/drm/gma500/psb_intel_sdvo.c
+++ b/drivers/gpu/drm/gma500/psb_intel_sdvo.c
@@ -406,18 +406,18 @@ static void psb_intel_sdvo_debug_write(struct psb_intel_sdvo *psb_intel_sdvo, u8
         DRM_DEBUG_KMS("%s: W: %02X ",
                                 SDVO_NAME(psb_intel_sdvo), cmd);
         for (i = 0; i < args_len; i++)
-                DRM_LOG_KMS("%02X ", ((u8 *)args)[i]);
+                DRM_DEBUG_KMS("%02X ", ((u8 *)args)[i]);
         for (; i < 8; i++)
-                DRM_LOG_KMS("   ");
+                DRM_DEBUG_KMS("   ");
         for (i = 0; i < ARRAY_SIZE(sdvo_cmd_names); i++) {
                 if (cmd == sdvo_cmd_names[i].cmd) {
-                        DRM_LOG_KMS("(%s)", sdvo_cmd_names[i].name);
+                        DRM_DEBUG_KMS("(%s)", sdvo_cmd_names[i].name);
                         break;
                 }
         }
         if (i == ARRAY_SIZE(sdvo_cmd_names))
-                DRM_LOG_KMS("(%02X)", cmd);
-        DRM_LOG_KMS("\n");
+                DRM_DEBUG_KMS("(%02X)", cmd);
+        DRM_DEBUG_KMS("\n");
 }
 
 static const char *cmd_status_names[] = {
@@ -512,9 +512,9 @@ static bool psb_intel_sdvo_read_response(struct psb_intel_sdvo *psb_intel_sdvo,
         }
 
         if (status <= SDVO_CMD_STATUS_SCALING_NOT_SUPP)
-                DRM_LOG_KMS("(%s)", cmd_status_names[status]);
+                DRM_DEBUG_KMS("(%s)", cmd_status_names[status]);
         else
-                DRM_LOG_KMS("(??? %d)", status);
+                DRM_DEBUG_KMS("(??? %d)", status);
 
         if (status != SDVO_CMD_STATUS_SUCCESS)
                 goto log_fail;
@@ -525,13 +525,13 @@ static bool psb_intel_sdvo_read_response(struct psb_intel_sdvo *psb_intel_sdvo,
                                           SDVO_I2C_RETURN_0 + i,
                                           &((u8 *)response)[i]))
                         goto log_fail;
-                DRM_LOG_KMS(" %02X", ((u8 *)response)[i]);
+                DRM_DEBUG_KMS(" %02X", ((u8 *)response)[i]);
         }
-        DRM_LOG_KMS("\n");
+        DRM_DEBUG_KMS("\n");
         return true;
 
 log_fail:
-        DRM_LOG_KMS("... failed\n");
+        DRM_DEBUG_KMS("... failed\n");
         return false;
 }
 
@@ -1844,7 +1844,7 @@ done:
         if (psb_intel_sdvo->base.base.crtc) {
                 struct drm_crtc *crtc = psb_intel_sdvo->base.base.crtc;
                 drm_crtc_helper_set_mode(crtc, &crtc->mode, crtc->x,
-                                         crtc->y, crtc->fb);
+                                         crtc->y, crtc->primary->fb);
         }
 
         return 0;
diff --git a/drivers/gpu/drm/gma500/psb_irq.c b/drivers/gpu/drm/gma500/psb_irq.c
index f883f9e4c524..624eb36511c5 100644
--- a/drivers/gpu/drm/gma500/psb_irq.c
+++ b/drivers/gpu/drm/gma500/psb_irq.c
@@ -200,11 +200,64 @@ static void psb_vdc_interrupt(struct drm_device *dev, uint32_t vdc_stat)
                 mid_pipe_event_handler(dev, 1);
 }
 
+/*
+ * SGX interrupt handler
+ */
+static void psb_sgx_interrupt(struct drm_device *dev, u32 stat_1, u32 stat_2)
+{
+        struct drm_psb_private *dev_priv = dev->dev_private;
+        u32 val, addr;
+        int error = false;
+
+        if (stat_1 & _PSB_CE_TWOD_COMPLETE)
+                val = PSB_RSGX32(PSB_CR_2D_BLIT_STATUS);
+
+        if (stat_2 & _PSB_CE2_BIF_REQUESTER_FAULT) {
+                val = PSB_RSGX32(PSB_CR_BIF_INT_STAT);
+                addr = PSB_RSGX32(PSB_CR_BIF_FAULT);
+                if (val) {
+                        if (val & _PSB_CBI_STAT_PF_N_RW)
+                                DRM_ERROR("SGX MMU page fault:");
+                        else
+                                DRM_ERROR("SGX MMU read / write protection fault:");
+
+                        if (val & _PSB_CBI_STAT_FAULT_CACHE)
+                                DRM_ERROR("\tCache requestor");
+                        if (val & _PSB_CBI_STAT_FAULT_TA)
+                                DRM_ERROR("\tTA requestor");
+                        if (val & _PSB_CBI_STAT_FAULT_VDM)
+                                DRM_ERROR("\tVDM requestor");
+                        if (val & _PSB_CBI_STAT_FAULT_2D)
+                                DRM_ERROR("\t2D requestor");
+                        if (val & _PSB_CBI_STAT_FAULT_PBE)
+                                DRM_ERROR("\tPBE requestor");
+                        if (val & _PSB_CBI_STAT_FAULT_TSP)
+                                DRM_ERROR("\tTSP requestor");
+                        if (val & _PSB_CBI_STAT_FAULT_ISP)
+                                DRM_ERROR("\tISP requestor");
+                        if (val & _PSB_CBI_STAT_FAULT_USSEPDS)
+                                DRM_ERROR("\tUSSEPDS requestor");
+                        if (val & _PSB_CBI_STAT_FAULT_HOST)
+                                DRM_ERROR("\tHost requestor");
+
+                        DRM_ERROR("\tMMU failing address is 0x%08x.\n",
+                                  (unsigned int)addr);
+                        error = true;
+                }
+        }
+
+        /* Clear bits */
+        PSB_WSGX32(stat_1, PSB_CR_EVENT_HOST_CLEAR);
+        PSB_WSGX32(stat_2, PSB_CR_EVENT_HOST_CLEAR2);
+        PSB_RSGX32(PSB_CR_EVENT_HOST_CLEAR2);
+}
+
 irqreturn_t psb_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = arg;
         struct drm_psb_private *dev_priv = dev->dev_private;
         uint32_t vdc_stat, dsp_int = 0, sgx_int = 0, hotplug_int = 0;
+        u32 sgx_stat_1, sgx_stat_2;
         int handled = 0;
 
         spin_lock(&dev_priv->irqmask_lock);
@@ -233,14 +286,9 @@ irqreturn_t psb_irq_handler(int irq, void *arg)
         }
 
         if (sgx_int) {
-                /* Not expected - we have it masked, shut it up */
-                u32 s, s2;
-                s = PSB_RSGX32(PSB_CR_EVENT_STATUS);
-                s2 = PSB_RSGX32(PSB_CR_EVENT_STATUS2);
-                PSB_WSGX32(s, PSB_CR_EVENT_HOST_CLEAR);
-                PSB_WSGX32(s2, PSB_CR_EVENT_HOST_CLEAR2);
-                /* if s & _PSB_CE_TWOD_COMPLETE we have 2D done but
-                   we may as well poll even if we add that ! */
+                sgx_stat_1 = PSB_RSGX32(PSB_CR_EVENT_STATUS);
+                sgx_stat_2 = PSB_RSGX32(PSB_CR_EVENT_STATUS2);
+                psb_sgx_interrupt(dev, sgx_stat_1, sgx_stat_2);
                 handled = 1;
         }
 
@@ -269,8 +317,13 @@ void psb_irq_preinstall(struct drm_device *dev)
 
         spin_lock_irqsave(&dev_priv->irqmask_lock, irqflags);
 
-        if (gma_power_is_on(dev))
+        if (gma_power_is_on(dev)) {
                 PSB_WVDC32(0xFFFFFFFF, PSB_HWSTAM);
+                PSB_WVDC32(0x00000000, PSB_INT_MASK_R);
+                PSB_WVDC32(0x00000000, PSB_INT_ENABLE_R);
+                PSB_WSGX32(0x00000000, PSB_CR_EVENT_HOST_ENABLE);
+                PSB_RSGX32(PSB_CR_EVENT_HOST_ENABLE);
+        }
         if (dev->vblank[0].enabled)
                 dev_priv->vdc_irq_mask |= _PSB_VSYNC_PIPEA_FLAG;
         if (dev->vblank[1].enabled)
@@ -286,7 +339,7 @@ void psb_irq_preinstall(struct drm_device *dev)
         /* Revisit this area - want per device masks ? */
         if (dev_priv->ops->hotplug)
                 dev_priv->vdc_irq_mask |= _PSB_IRQ_DISP_HOTSYNC;
-        dev_priv->vdc_irq_mask |= _PSB_IRQ_ASLE;
+        dev_priv->vdc_irq_mask |= _PSB_IRQ_ASLE | _PSB_IRQ_SGX_FLAG;
 
         /* This register is safe even if display island is off */
         PSB_WVDC32(~dev_priv->vdc_irq_mask, PSB_INT_MASK_R);
@@ -295,12 +348,16 @@ void psb_irq_preinstall(struct drm_device *dev)
 
 int psb_irq_postinstall(struct drm_device *dev)
 {
-        struct drm_psb_private *dev_priv =
-            (struct drm_psb_private *) dev->dev_private;
+        struct drm_psb_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
 
         spin_lock_irqsave(&dev_priv->irqmask_lock, irqflags);
 
+        /* Enable 2D and MMU fault interrupts */
+        PSB_WSGX32(_PSB_CE2_BIF_REQUESTER_FAULT, PSB_CR_EVENT_HOST_ENABLE2);
+        PSB_WSGX32(_PSB_CE_TWOD_COMPLETE, PSB_CR_EVENT_HOST_ENABLE);
+        PSB_RSGX32(PSB_CR_EVENT_HOST_ENABLE); /* Post */
+
         /* This register is safe even if display island is off */
         PSB_WVDC32(dev_priv->vdc_irq_mask, PSB_INT_ENABLE_R);
         PSB_WVDC32(0xFFFFFFFF, PSB_HWSTAM);
diff --git a/drivers/gpu/drm/i2c/tda998x_drv.c b/drivers/gpu/drm/i2c/tda998x_drv.c
index faa77f543a07..48af5cac1902 100644
--- a/drivers/gpu/drm/i2c/tda998x_drv.c
+++ b/drivers/gpu/drm/i2c/tda998x_drv.c
@@ -19,6 +19,8 @@
 
 #include <linux/hdmi.h>
 #include <linux/module.h>
+#include <linux/irq.h>
+#include <sound/asoundef.h>
 
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
@@ -30,6 +32,7 @@
 
 struct tda998x_priv {
         struct i2c_client *cec;
+        struct i2c_client *hdmi;
         uint16_t rev;
         uint8_t current_page;
         int dpms;
@@ -38,6 +41,10 @@ struct tda998x_priv {
         u8 vip_cntrl_1;
         u8 vip_cntrl_2;
         struct tda998x_encoder_params params;
+
+        wait_queue_head_t wq_edid;
+        volatile int wq_edid_wait;
+        struct drm_encoder *encoder;
 };
 
 #define to_tda998x_priv(x)  ((struct tda998x_priv *)to_encoder_slave(x)->slave_priv)
@@ -120,6 +127,8 @@ struct tda998x_priv {
 # define VIP_CNTRL_5_CKCASE       (1 << 0)
 # define VIP_CNTRL_5_SP_CNT(x)    (((x) & 3) << 1)
 #define REG_MUX_AP                REG(0x00, 0x26)     /* read/write */
+# define MUX_AP_SELECT_I2S        0x64
+# define MUX_AP_SELECT_SPDIF      0x40
 #define REG_MUX_VP_VIP_OUT        REG(0x00, 0x27)     /* read/write */
 #define REG_MAT_CONTRL            REG(0x00, 0x80)     /* write */
 # define MAT_CONTRL_MAT_SC(x)     (((x) & 3) << 0)
@@ -197,10 +206,11 @@ struct tda998x_priv {
 #define REG_I2S_FORMAT            REG(0x00, 0xfc)     /* read/write */
 # define I2S_FORMAT(x)            (((x) & 3) << 0)
 #define REG_AIP_CLKSEL            REG(0x00, 0xfd)     /* write */
-# define AIP_CLKSEL_FS(x)         (((x) & 3) << 0)
-# define AIP_CLKSEL_CLK_POL(x)    (((x) & 1) << 2)
-# define AIP_CLKSEL_AIP(x)        (((x) & 7) << 3)
-
+# define AIP_CLKSEL_AIP_SPDIF     (0 << 3)
+# define AIP_CLKSEL_AIP_I2S       (1 << 3)
+# define AIP_CLKSEL_FS_ACLK       (0 << 0)
+# define AIP_CLKSEL_FS_MCLK       (1 << 0)
+# define AIP_CLKSEL_FS_FS64SPDIF  (2 << 0)
 
 /* Page 02h: PLL settings */
 #define REG_PLL_SERIAL_1          REG(0x02, 0x00)     /* read/write */
@@ -304,11 +314,16 @@ struct tda998x_priv {
 
 /* CEC registers: (not paged)
  */
+#define REG_CEC_INTSTATUS         0xee                /* read */
+# define CEC_INTSTATUS_CEC        (1 << 0)
+# define CEC_INTSTATUS_HDMI       (1 << 1)
 #define REG_CEC_FRO_IM_CLK_CTRL   0xfb                /* read/write */
 # define CEC_FRO_IM_CLK_CTRL_GHOST_DIS (1 << 7)
 # define CEC_FRO_IM_CLK_CTRL_ENA_OTP   (1 << 6)
 # define CEC_FRO_IM_CLK_CTRL_IMCLK_SEL (1 << 1)
 # define CEC_FRO_IM_CLK_CTRL_FRO_DIV   (1 << 0)
+#define REG_CEC_RXSHPDINTENA      0xfc                /* read/write */
+#define REG_CEC_RXSHPDINT         0xfd                /* read */
 #define REG_CEC_RXSHPDLEV         0xfe                /* read */
 # define CEC_RXSHPDLEV_RXSENS     (1 << 0)
 # define CEC_RXSHPDLEV_HPD        (1 << 1)
@@ -328,21 +343,21 @@ struct tda998x_priv {
 #define TDA19988                  0x0301
 
 static void
-cec_write(struct drm_encoder *encoder, uint16_t addr, uint8_t val)
+cec_write(struct tda998x_priv *priv, uint16_t addr, uint8_t val)
 {
-        struct i2c_client *client = to_tda998x_priv(encoder)->cec;
+        struct i2c_client *client = priv->cec;
         uint8_t buf[] = {addr, val};
         int ret;
 
-        ret = i2c_master_send(client, buf, ARRAY_SIZE(buf));
+        ret = i2c_master_send(client, buf, sizeof(buf));
         if (ret < 0)
                 dev_err(&client->dev, "Error %d writing to cec:0x%x\n", ret, addr);
 }
 
 static uint8_t
-cec_read(struct drm_encoder *encoder, uint8_t addr)
+cec_read(struct tda998x_priv *priv, uint8_t addr)
 {
-        struct i2c_client *client = to_tda998x_priv(encoder)->cec;
+        struct i2c_client *client = priv->cec;
         uint8_t val;
         int ret;
 
@@ -361,32 +376,36 @@ fail:
         return 0;
 }
 
-static void
-set_page(struct drm_encoder *encoder, uint16_t reg)
+static int
+set_page(struct tda998x_priv *priv, uint16_t reg)
 {
-        struct tda998x_priv *priv = to_tda998x_priv(encoder);
-
         if (REG2PAGE(reg) != priv->current_page) {
-                struct i2c_client *client = drm_i2c_encoder_get_client(encoder);
+                struct i2c_client *client = priv->hdmi;
                 uint8_t buf[] = {
                                 REG_CURPAGE, REG2PAGE(reg)
                 };
                 int ret = i2c_master_send(client, buf, sizeof(buf));
-                if (ret < 0)
-                        dev_err(&client->dev, "Error %d writing to REG_CURPAGE\n", ret);
+                if (ret < 0) {
+                        dev_err(&client->dev, "setpage %04x err %d\n",
+                                        reg, ret);
+                        return ret;
+                }
 
                 priv->current_page = REG2PAGE(reg);
         }
+        return 0;
 }
 
 static int
-reg_read_range(struct drm_encoder *encoder, uint16_t reg, char *buf, int cnt)
+reg_read_range(struct tda998x_priv *priv, uint16_t reg, char *buf, int cnt)
 {
-        struct i2c_client *client = drm_i2c_encoder_get_client(encoder);
+        struct i2c_client *client = priv->hdmi;
         uint8_t addr = REG2ADDR(reg);
         int ret;
 
-        set_page(encoder, reg);
+        ret = set_page(priv, reg);
+        if (ret < 0)
+                return ret;
 
         ret = i2c_master_send(client, &addr, sizeof(addr));
         if (ret < 0)
@@ -404,100 +423,147 @@ fail:
 }
 
 static void
-reg_write_range(struct drm_encoder *encoder, uint16_t reg, uint8_t *p, int cnt)
+reg_write_range(struct tda998x_priv *priv, uint16_t reg, uint8_t *p, int cnt)
 {
-        struct i2c_client *client = drm_i2c_encoder_get_client(encoder);
+        struct i2c_client *client = priv->hdmi;
         uint8_t buf[cnt+1];
         int ret;
 
         buf[0] = REG2ADDR(reg);
         memcpy(&buf[1], p, cnt);
 
-        set_page(encoder, reg);
+        ret = set_page(priv, reg);
+        if (ret < 0)
+                return;
 
         ret = i2c_master_send(client, buf, cnt + 1);
         if (ret < 0)
                 dev_err(&client->dev, "Error %d writing to 0x%x\n", ret, reg);
 }
 
-static uint8_t
-reg_read(struct drm_encoder *encoder, uint16_t reg)
+static int
+reg_read(struct tda998x_priv *priv, uint16_t reg)
 {
         uint8_t val = 0;
-        reg_read_range(encoder, reg, &val, sizeof(val));
+        int ret;
+
+        ret = reg_read_range(priv, reg, &val, sizeof(val));
+        if (ret < 0)
+                return ret;
         return val;
 }
 
 static void
-reg_write(struct drm_encoder *encoder, uint16_t reg, uint8_t val)
+reg_write(struct tda998x_priv *priv, uint16_t reg, uint8_t val)
 {
-        struct i2c_client *client = drm_i2c_encoder_get_client(encoder);
+        struct i2c_client *client = priv->hdmi;
         uint8_t buf[] = {REG2ADDR(reg), val};
         int ret;
 
-        set_page(encoder, reg);
+        ret = set_page(priv, reg);
+        if (ret < 0)
+                return;
 
-        ret = i2c_master_send(client, buf, ARRAY_SIZE(buf));
+        ret = i2c_master_send(client, buf, sizeof(buf));
         if (ret < 0)
                 dev_err(&client->dev, "Error %d writing to 0x%x\n", ret, reg);
 }
 
 static void
-reg_write16(struct drm_encoder *encoder, uint16_t reg, uint16_t val)
+reg_write16(struct tda998x_priv *priv, uint16_t reg, uint16_t val)
 {
-        struct i2c_client *client = drm_i2c_encoder_get_client(encoder);
+        struct i2c_client *client = priv->hdmi;
         uint8_t buf[] = {REG2ADDR(reg), val >> 8, val};
         int ret;
 
-        set_page(encoder, reg);
+        ret = set_page(priv, reg);
+        if (ret < 0)
+                return;
 
-        ret = i2c_master_send(client, buf, ARRAY_SIZE(buf));
+        ret = i2c_master_send(client, buf, sizeof(buf));
         if (ret < 0)
                 dev_err(&client->dev, "Error %d writing to 0x%x\n", ret, reg);
 }
 
 static void
-reg_set(struct drm_encoder *encoder, uint16_t reg, uint8_t val)
+reg_set(struct tda998x_priv *priv, uint16_t reg, uint8_t val)
 {
-        reg_write(encoder, reg, reg_read(encoder, reg) | val);
+        int old_val;
+
+        old_val = reg_read(priv, reg);
+        if (old_val >= 0)
+                reg_write(priv, reg, old_val | val);
 }
 
 static void
-reg_clear(struct drm_encoder *encoder, uint16_t reg, uint8_t val)
+reg_clear(struct tda998x_priv *priv, uint16_t reg, uint8_t val)
 {
-        reg_write(encoder, reg, reg_read(encoder, reg) & ~val);
+        int old_val;
+
+        old_val = reg_read(priv, reg);
+        if (old_val >= 0)
+                reg_write(priv, reg, old_val & ~val);
 }
 
 static void
-tda998x_reset(struct drm_encoder *encoder)
+tda998x_reset(struct tda998x_priv *priv)
 {
         /* reset audio and i2c master: */
-        reg_set(encoder, REG_SOFTRESET, SOFTRESET_AUDIO | SOFTRESET_I2C_MASTER);
+        reg_write(priv, REG_SOFTRESET, SOFTRESET_AUDIO | SOFTRESET_I2C_MASTER);
         msleep(50);
-        reg_clear(encoder, REG_SOFTRESET, SOFTRESET_AUDIO | SOFTRESET_I2C_MASTER);
+        reg_write(priv, REG_SOFTRESET, 0);
         msleep(50);
 
         /* reset transmitter: */
-        reg_set(encoder, REG_MAIN_CNTRL0, MAIN_CNTRL0_SR);
-        reg_clear(encoder, REG_MAIN_CNTRL0, MAIN_CNTRL0_SR);
+        reg_set(priv, REG_MAIN_CNTRL0, MAIN_CNTRL0_SR);
+        reg_clear(priv, REG_MAIN_CNTRL0, MAIN_CNTRL0_SR);
 
         /* PLL registers common configuration */
-        reg_write(encoder, REG_PLL_SERIAL_1, 0x00);
-        reg_write(encoder, REG_PLL_SERIAL_2, PLL_SERIAL_2_SRL_NOSC(1));
-        reg_write(encoder, REG_PLL_SERIAL_3, 0x00);
-        reg_write(encoder, REG_SERIALIZER,   0x00);
-        reg_write(encoder, REG_BUFFER_OUT,   0x00);
-        reg_write(encoder, REG_PLL_SCG1,     0x00);
-        reg_write(encoder, REG_AUDIO_DIV,    AUDIO_DIV_SERCLK_8);
-        reg_write(encoder, REG_SEL_CLK,      SEL_CLK_SEL_CLK1 | SEL_CLK_ENA_SC_CLK);
-        reg_write(encoder, REG_PLL_SCGN1,    0xfa);
-        reg_write(encoder, REG_PLL_SCGN2,    0x00);
-        reg_write(encoder, REG_PLL_SCGR1,    0x5b);
-        reg_write(encoder, REG_PLL_SCGR2,    0x00);
-        reg_write(encoder, REG_PLL_SCG2,     0x10);
+        reg_write(priv, REG_PLL_SERIAL_1, 0x00);
+        reg_write(priv, REG_PLL_SERIAL_2, PLL_SERIAL_2_SRL_NOSC(1));
+        reg_write(priv, REG_PLL_SERIAL_3, 0x00);
+        reg_write(priv, REG_SERIALIZER,   0x00);
+        reg_write(priv, REG_BUFFER_OUT,   0x00);
+        reg_write(priv, REG_PLL_SCG1,     0x00);
+        reg_write(priv, REG_AUDIO_DIV,    AUDIO_DIV_SERCLK_8);
+        reg_write(priv, REG_SEL_CLK,      SEL_CLK_SEL_CLK1 | SEL_CLK_ENA_SC_CLK);
+        reg_write(priv, REG_PLL_SCGN1,    0xfa);
+        reg_write(priv, REG_PLL_SCGN2,    0x00);
+        reg_write(priv, REG_PLL_SCGR1,    0x5b);
+        reg_write(priv, REG_PLL_SCGR2,    0x00);
+        reg_write(priv, REG_PLL_SCG2,     0x10);
 
         /* Write the default value MUX register */
-        reg_write(encoder, REG_MUX_VP_VIP_OUT, 0x24);
+        reg_write(priv, REG_MUX_VP_VIP_OUT, 0x24);
+}
+
+/*
+ * only 2 interrupts may occur: screen plug/unplug and EDID read
+ */
+static irqreturn_t tda998x_irq_thread(int irq, void *data)
+{
+        struct tda998x_priv *priv = data;
+        u8 sta, cec, lvl, flag0, flag1, flag2;
+
+        if (!priv)
+                return IRQ_HANDLED;
+        sta = cec_read(priv, REG_CEC_INTSTATUS);
+        cec = cec_read(priv, REG_CEC_RXSHPDINT);
+        lvl = cec_read(priv, REG_CEC_RXSHPDLEV);
+        flag0 = reg_read(priv, REG_INT_FLAGS_0);
+        flag1 = reg_read(priv, REG_INT_FLAGS_1);
+        flag2 = reg_read(priv, REG_INT_FLAGS_2);
+        DRM_DEBUG_DRIVER(
+                "tda irq sta %02x cec %02x lvl %02x f0 %02x f1 %02x f2 %02x\n",
+                sta, cec, lvl, flag0, flag1, flag2);
+        if ((flag2 & INT_FLAGS_2_EDID_BLK_RD) && priv->wq_edid_wait) {
+                priv->wq_edid_wait = 0;
+                wake_up(&priv->wq_edid);
+        } else if (cec != 0) {                  /* HPD change */
+                if (priv->encoder && priv->encoder->dev)
+                        drm_helper_hpd_irq_event(priv->encoder->dev);
+        }
+        return IRQ_HANDLED;
 }
 
 static uint8_t tda998x_cksum(uint8_t *buf, size_t bytes)
@@ -513,91 +579,88 @@ static uint8_t tda998x_cksum(uint8_t *buf, size_t bytes)
 #define PB(x) (HB(2) + 1 + (x))
 
 static void
-tda998x_write_if(struct drm_encoder *encoder, uint8_t bit, uint16_t addr,
+tda998x_write_if(struct tda998x_priv *priv, uint8_t bit, uint16_t addr,
                  uint8_t *buf, size_t size)
 {
         buf[PB(0)] = tda998x_cksum(buf, size);
 
-        reg_clear(encoder, REG_DIP_IF_FLAGS, bit);
-        reg_write_range(encoder, addr, buf, size);
-        reg_set(encoder, REG_DIP_IF_FLAGS, bit);
+        reg_clear(priv, REG_DIP_IF_FLAGS, bit);
+        reg_write_range(priv, addr, buf, size);
+        reg_set(priv, REG_DIP_IF_FLAGS, bit);
 }
 
 static void
-tda998x_write_aif(struct drm_encoder *encoder, struct tda998x_encoder_params *p)
+tda998x_write_aif(struct tda998x_priv *priv, struct tda998x_encoder_params *p)
 {
-        uint8_t buf[PB(5) + 1];
+        u8 buf[PB(HDMI_AUDIO_INFOFRAME_SIZE) + 1];
 
         memset(buf, 0, sizeof(buf));
-        buf[HB(0)] = 0x84;
+        buf[HB(0)] = HDMI_INFOFRAME_TYPE_AUDIO;
         buf[HB(1)] = 0x01;
-        buf[HB(2)] = 10;
+        buf[HB(2)] = HDMI_AUDIO_INFOFRAME_SIZE;
         buf[PB(1)] = p->audio_frame[1] & 0x07; /* CC */
         buf[PB(2)] = p->audio_frame[2] & 0x1c; /* SF */
         buf[PB(4)] = p->audio_frame[4];
         buf[PB(5)] = p->audio_frame[5] & 0xf8; /* DM_INH + LSV */
 
-        tda998x_write_if(encoder, DIP_IF_FLAGS_IF4, REG_IF4_HB0, buf,
+        tda998x_write_if(priv, DIP_IF_FLAGS_IF4, REG_IF4_HB0, buf,
                          sizeof(buf));
 }
 
 static void
-tda998x_write_avi(struct drm_encoder *encoder, struct drm_display_mode *mode)
+tda998x_write_avi(struct tda998x_priv *priv, struct drm_display_mode *mode)
 {
-        uint8_t buf[PB(13) + 1];
+        u8 buf[PB(HDMI_AVI_INFOFRAME_SIZE) + 1];
 
         memset(buf, 0, sizeof(buf));
-        buf[HB(0)] = 0x82;
+        buf[HB(0)] = HDMI_INFOFRAME_TYPE_AVI;
         buf[HB(1)] = 0x02;
-        buf[HB(2)] = 13;
+        buf[HB(2)] = HDMI_AVI_INFOFRAME_SIZE;
         buf[PB(1)] = HDMI_SCAN_MODE_UNDERSCAN;
+        buf[PB(2)] = HDMI_ACTIVE_ASPECT_PICTURE;
         buf[PB(3)] = HDMI_QUANTIZATION_RANGE_FULL << 2;
         buf[PB(4)] = drm_match_cea_mode(mode);
 
-        tda998x_write_if(encoder, DIP_IF_FLAGS_IF2, REG_IF2_HB0, buf,
+        tda998x_write_if(priv, DIP_IF_FLAGS_IF2, REG_IF2_HB0, buf,
                          sizeof(buf));
 }
 
-static void tda998x_audio_mute(struct drm_encoder *encoder, bool on)
+static void tda998x_audio_mute(struct tda998x_priv *priv, bool on)
 {
         if (on) {
-                reg_set(encoder, REG_SOFTRESET, SOFTRESET_AUDIO);
-                reg_clear(encoder, REG_SOFTRESET, SOFTRESET_AUDIO);
-                reg_set(encoder, REG_AIP_CNTRL_0, AIP_CNTRL_0_RST_FIFO);
+                reg_set(priv, REG_SOFTRESET, SOFTRESET_AUDIO);
+                reg_clear(priv, REG_SOFTRESET, SOFTRESET_AUDIO);
+                reg_set(priv, REG_AIP_CNTRL_0, AIP_CNTRL_0_RST_FIFO);
         } else {
-                reg_clear(encoder, REG_AIP_CNTRL_0, AIP_CNTRL_0_RST_FIFO);
+                reg_clear(priv, REG_AIP_CNTRL_0, AIP_CNTRL_0_RST_FIFO);
         }
 }
 
 static void
-tda998x_configure_audio(struct drm_encoder *encoder,
+tda998x_configure_audio(struct tda998x_priv *priv,
                 struct drm_display_mode *mode, struct tda998x_encoder_params *p)
 {
-        uint8_t buf[6], clksel_aip, clksel_fs, ca_i2s, cts_n, adiv;
+        uint8_t buf[6], clksel_aip, clksel_fs, cts_n, adiv;
         uint32_t n;
 
         /* Enable audio ports */
-        reg_write(encoder, REG_ENA_AP, p->audio_cfg);
-        reg_write(encoder, REG_ENA_ACLK, p->audio_clk_cfg);
+        reg_write(priv, REG_ENA_AP, p->audio_cfg);
+        reg_write(priv, REG_ENA_ACLK, p->audio_clk_cfg);
 
         /* Set audio input source */
         switch (p->audio_format) {
         case AFMT_SPDIF:
-                reg_write(encoder, REG_MUX_AP, 0x40);
-                clksel_aip = AIP_CLKSEL_AIP(0);
-                /* FS64SPDIF */
-                clksel_fs = AIP_CLKSEL_FS(2);
+                reg_write(priv, REG_MUX_AP, MUX_AP_SELECT_SPDIF);
+                clksel_aip = AIP_CLKSEL_AIP_SPDIF;
+                clksel_fs = AIP_CLKSEL_FS_FS64SPDIF;
                 cts_n = CTS_N_M(3) | CTS_N_K(3);
-                ca_i2s = 0;
                 break;
 
         case AFMT_I2S:
-                reg_write(encoder, REG_MUX_AP, 0x64);
-                clksel_aip = AIP_CLKSEL_AIP(1);
-                /* ACLK */
-                clksel_fs = AIP_CLKSEL_FS(0);
+                reg_write(priv, REG_MUX_AP, MUX_AP_SELECT_I2S);
+                clksel_aip = AIP_CLKSEL_AIP_I2S;
+                clksel_fs = AIP_CLKSEL_FS_ACLK;
                 cts_n = CTS_N_M(3) | CTS_N_K(3);
-                ca_i2s = CA_I2S_CA_I2S(0);
                 break;
 
         default:
@@ -605,12 +668,10 @@ tda998x_configure_audio(struct drm_encoder *encoder,
                 return;
         }
 
-        reg_write(encoder, REG_AIP_CLKSEL, clksel_aip);
-        reg_clear(encoder, REG_AIP_CNTRL_0, AIP_CNTRL_0_LAYOUT);
-
-        /* Enable automatic CTS generation */
-        reg_clear(encoder, REG_AIP_CNTRL_0, AIP_CNTRL_0_ACR_MAN);
-        reg_write(encoder, REG_CTS_N, cts_n);
+        reg_write(priv, REG_AIP_CLKSEL, clksel_aip);
+        reg_clear(priv, REG_AIP_CNTRL_0, AIP_CNTRL_0_LAYOUT |
+                                        AIP_CNTRL_0_ACR_MAN);   /* auto CTS */
+        reg_write(priv, REG_CTS_N, cts_n);
 
         /*
          * Audio input somehow depends on HDMI line rate which is
@@ -619,11 +680,15 @@ tda998x_configure_audio(struct drm_encoder *encoder,
          * There is no detailed info in the datasheet, so we just
          * assume 100MHz requires larger divider.
          */
+        adiv = AUDIO_DIV_SERCLK_8;
         if (mode->clock > 100000)
-                adiv = AUDIO_DIV_SERCLK_16;
-        else
-                adiv = AUDIO_DIV_SERCLK_8;
-        reg_write(encoder, REG_AUDIO_DIV, adiv);
+                adiv++;                 /* AUDIO_DIV_SERCLK_16 */
+
+        /* S/PDIF asks for a larger divider */
+        if (p->audio_format == AFMT_SPDIF)
+                adiv++;                 /* AUDIO_DIV_SERCLK_16 or _32 */
+
+        reg_write(priv, REG_AUDIO_DIV, adiv);
 
         /*
          * This is the approximate value of N, which happens to be
@@ -638,28 +703,29 @@ tda998x_configure_audio(struct drm_encoder *encoder,
         buf[3] = n;
         buf[4] = n >> 8;
         buf[5] = n >> 16;
-        reg_write_range(encoder, REG_ACR_CTS_0, buf, 6);
+        reg_write_range(priv, REG_ACR_CTS_0, buf, 6);
 
         /* Set CTS clock reference */
-        reg_write(encoder, REG_AIP_CLKSEL, clksel_aip | clksel_fs);
+        reg_write(priv, REG_AIP_CLKSEL, clksel_aip | clksel_fs);
 
         /* Reset CTS generator */
-        reg_set(encoder, REG_AIP_CNTRL_0, AIP_CNTRL_0_RST_CTS);
-        reg_clear(encoder, REG_AIP_CNTRL_0, AIP_CNTRL_0_RST_CTS);
+        reg_set(priv, REG_AIP_CNTRL_0, AIP_CNTRL_0_RST_CTS);
+        reg_clear(priv, REG_AIP_CNTRL_0, AIP_CNTRL_0_RST_CTS);
 
         /* Write the channel status */
-        buf[0] = 0x04;
+        buf[0] = IEC958_AES0_CON_NOT_COPYRIGHT;
         buf[1] = 0x00;
-        buf[2] = 0x00;
-        buf[3] = 0xf1;
-        reg_write_range(encoder, REG_CH_STAT_B(0), buf, 4);
+        buf[2] = IEC958_AES3_CON_FS_NOTID;
+        buf[3] = IEC958_AES4_CON_ORIGFS_NOTID |
+                        IEC958_AES4_CON_MAX_WORDLEN_24;
+        reg_write_range(priv, REG_CH_STAT_B(0), buf, 4);
 
-        tda998x_audio_mute(encoder, true);
-        mdelay(20);
-        tda998x_audio_mute(encoder, false);
+        tda998x_audio_mute(priv, true);
+        msleep(20);
+        tda998x_audio_mute(priv, false);
 
         /* Write the audio information packet */
-        tda998x_write_aif(encoder, p);
+        tda998x_write_aif(priv, p);
 }
 
 /* DRM encoder functions */
@@ -701,19 +767,19 @@ tda998x_encoder_dpms(struct drm_encoder *encoder, int mode)
         switch (mode) {
         case DRM_MODE_DPMS_ON:
                 /* enable video ports, audio will be enabled later */
-                reg_write(encoder, REG_ENA_VP_0, 0xff);
-                reg_write(encoder, REG_ENA_VP_1, 0xff);
-                reg_write(encoder, REG_ENA_VP_2, 0xff);
+                reg_write(priv, REG_ENA_VP_0, 0xff);
+                reg_write(priv, REG_ENA_VP_1, 0xff);
+                reg_write(priv, REG_ENA_VP_2, 0xff);
                 /* set muxing after enabling ports: */
-                reg_write(encoder, REG_VIP_CNTRL_0, priv->vip_cntrl_0);
-                reg_write(encoder, REG_VIP_CNTRL_1, priv->vip_cntrl_1);
-                reg_write(encoder, REG_VIP_CNTRL_2, priv->vip_cntrl_2);
+                reg_write(priv, REG_VIP_CNTRL_0, priv->vip_cntrl_0);
+                reg_write(priv, REG_VIP_CNTRL_1, priv->vip_cntrl_1);
+                reg_write(priv, REG_VIP_CNTRL_2, priv->vip_cntrl_2);
                 break;
         case DRM_MODE_DPMS_OFF:
                 /* disable video ports */
-                reg_write(encoder, REG_ENA_VP_0, 0x00);
-                reg_write(encoder, REG_ENA_VP_1, 0x00);
-                reg_write(encoder, REG_ENA_VP_2, 0x00);
+                reg_write(priv, REG_ENA_VP_0, 0x00);
+                reg_write(priv, REG_ENA_VP_1, 0x00);
+                reg_write(priv, REG_ENA_VP_2, 0x00);
                 break;
         }
 
@@ -831,110 +897,110 @@ tda998x_encoder_mode_set(struct drm_encoder *encoder,
         }
 
         /* mute the audio FIFO: */
-        reg_set(encoder, REG_AIP_CNTRL_0, AIP_CNTRL_0_RST_FIFO);
+        reg_set(priv, REG_AIP_CNTRL_0, AIP_CNTRL_0_RST_FIFO);
 
         /* set HDMI HDCP mode off: */
-        reg_set(encoder, REG_TBG_CNTRL_1, TBG_CNTRL_1_DWIN_DIS);
-        reg_clear(encoder, REG_TX33, TX33_HDMI);
+        reg_write(priv, REG_TBG_CNTRL_1, TBG_CNTRL_1_DWIN_DIS);
+        reg_clear(priv, REG_TX33, TX33_HDMI);
+        reg_write(priv, REG_ENC_CNTRL, ENC_CNTRL_CTL_CODE(0));
 
-        reg_write(encoder, REG_ENC_CNTRL, ENC_CNTRL_CTL_CODE(0));
         /* no pre-filter or interpolator: */
-        reg_write(encoder, REG_HVF_CNTRL_0, HVF_CNTRL_0_PREFIL(0) |
+        reg_write(priv, REG_HVF_CNTRL_0, HVF_CNTRL_0_PREFIL(0) |
                         HVF_CNTRL_0_INTPOL(0));
-        reg_write(encoder, REG_VIP_CNTRL_5, VIP_CNTRL_5_SP_CNT(0));
-        reg_write(encoder, REG_VIP_CNTRL_4, VIP_CNTRL_4_BLANKIT(0) |
+        reg_write(priv, REG_VIP_CNTRL_5, VIP_CNTRL_5_SP_CNT(0));
+        reg_write(priv, REG_VIP_CNTRL_4, VIP_CNTRL_4_BLANKIT(0) |
                         VIP_CNTRL_4_BLC(0));
-        reg_clear(encoder, REG_PLL_SERIAL_3, PLL_SERIAL_3_SRL_CCIR);
 
-        reg_clear(encoder, REG_PLL_SERIAL_1, PLL_SERIAL_1_SRL_MAN_IZ);
-        reg_clear(encoder, REG_PLL_SERIAL_3, PLL_SERIAL_3_SRL_DE);
-        reg_write(encoder, REG_SERIALIZER, 0);
-        reg_write(encoder, REG_HVF_CNTRL_1, HVF_CNTRL_1_VQR(0));
+        reg_clear(priv, REG_PLL_SERIAL_1, PLL_SERIAL_1_SRL_MAN_IZ);
+        reg_clear(priv, REG_PLL_SERIAL_3, PLL_SERIAL_3_SRL_CCIR |
+                                          PLL_SERIAL_3_SRL_DE);
+        reg_write(priv, REG_SERIALIZER, 0);
+        reg_write(priv, REG_HVF_CNTRL_1, HVF_CNTRL_1_VQR(0));
 
         /* TODO enable pixel repeat for pixel rates less than 25Msamp/s */
         rep = 0;
-        reg_write(encoder, REG_RPT_CNTRL, 0);
-        reg_write(encoder, REG_SEL_CLK, SEL_CLK_SEL_VRF_CLK(0) |
+        reg_write(priv, REG_RPT_CNTRL, 0);
+        reg_write(priv, REG_SEL_CLK, SEL_CLK_SEL_VRF_CLK(0) |
                         SEL_CLK_SEL_CLK1 | SEL_CLK_ENA_SC_CLK);
 
-        reg_write(encoder, REG_PLL_SERIAL_2, PLL_SERIAL_2_SRL_NOSC(div) |
+        reg_write(priv, REG_PLL_SERIAL_2, PLL_SERIAL_2_SRL_NOSC(div) |
                         PLL_SERIAL_2_SRL_PR(rep));
 
         /* set color matrix bypass flag: */
-        reg_set(encoder, REG_MAT_CONTRL, MAT_CONTRL_MAT_BP);
+        reg_write(priv, REG_MAT_CONTRL, MAT_CONTRL_MAT_BP |
+                                MAT_CONTRL_MAT_SC(1));
 
         /* set BIAS tmds value: */
-        reg_write(encoder, REG_ANA_GENERAL, 0x09);
-
-        reg_clear(encoder, REG_TBG_CNTRL_0, TBG_CNTRL_0_SYNC_MTHD);
+        reg_write(priv, REG_ANA_GENERAL, 0x09);
 
         /*
          * Sync on rising HSYNC/VSYNC
          */
-        reg_write(encoder, REG_VIP_CNTRL_3, 0);
-        reg_set(encoder, REG_VIP_CNTRL_3, VIP_CNTRL_3_SYNC_HS);
+        reg = VIP_CNTRL_3_SYNC_HS;
 
         /*
          * TDA19988 requires high-active sync at input stage,
          * so invert low-active sync provided by master encoder here
          */
         if (mode->flags & DRM_MODE_FLAG_NHSYNC)
-                reg_set(encoder, REG_VIP_CNTRL_3, VIP_CNTRL_3_H_TGL);
+                reg |= VIP_CNTRL_3_H_TGL;
         if (mode->flags & DRM_MODE_FLAG_NVSYNC)
-                reg_set(encoder, REG_VIP_CNTRL_3, VIP_CNTRL_3_V_TGL);
+                reg |= VIP_CNTRL_3_V_TGL;
+        reg_write(priv, REG_VIP_CNTRL_3, reg);
+
+        reg_write(priv, REG_VIDFORMAT, 0x00);
+        reg_write16(priv, REG_REFPIX_MSB, ref_pix);
+        reg_write16(priv, REG_REFLINE_MSB, ref_line);
+        reg_write16(priv, REG_NPIX_MSB, n_pix);
+        reg_write16(priv, REG_NLINE_MSB, n_line);
+        reg_write16(priv, REG_VS_LINE_STRT_1_MSB, vs1_line_s);
+        reg_write16(priv, REG_VS_PIX_STRT_1_MSB, vs1_pix_s);
+        reg_write16(priv, REG_VS_LINE_END_1_MSB, vs1_line_e);
+        reg_write16(priv, REG_VS_PIX_END_1_MSB, vs1_pix_e);
+        reg_write16(priv, REG_VS_LINE_STRT_2_MSB, vs2_line_s);
+        reg_write16(priv, REG_VS_PIX_STRT_2_MSB, vs2_pix_s);
+        reg_write16(priv, REG_VS_LINE_END_2_MSB, vs2_line_e);
+        reg_write16(priv, REG_VS_PIX_END_2_MSB, vs2_pix_e);
+        reg_write16(priv, REG_HS_PIX_START_MSB, hs_pix_s);
+        reg_write16(priv, REG_HS_PIX_STOP_MSB, hs_pix_e);
+        reg_write16(priv, REG_VWIN_START_1_MSB, vwin1_line_s);
+        reg_write16(priv, REG_VWIN_END_1_MSB, vwin1_line_e);
+        reg_write16(priv, REG_VWIN_START_2_MSB, vwin2_line_s);
+        reg_write16(priv, REG_VWIN_END_2_MSB, vwin2_line_e);
+        reg_write16(priv, REG_DE_START_MSB, de_pix_s);
+        reg_write16(priv, REG_DE_STOP_MSB, de_pix_e);
+
+        if (priv->rev == TDA19988) {
+                /* let incoming pixels fill the active space (if any) */
+                reg_write(priv, REG_ENABLE_SPACE, 0x00);
+        }
 
         /*
          * Always generate sync polarity relative to input sync and
          * revert input stage toggled sync at output stage
          */
-        reg = TBG_CNTRL_1_TGL_EN;
+        reg = TBG_CNTRL_1_DWIN_DIS | TBG_CNTRL_1_TGL_EN;
         if (mode->flags & DRM_MODE_FLAG_NHSYNC)
                 reg |= TBG_CNTRL_1_H_TGL;
         if (mode->flags & DRM_MODE_FLAG_NVSYNC)
                 reg |= TBG_CNTRL_1_V_TGL;
-        reg_write(encoder, REG_TBG_CNTRL_1, reg);
-
-        reg_write(encoder, REG_VIDFORMAT, 0x00);
-        reg_write16(encoder, REG_REFPIX_MSB, ref_pix);
-        reg_write16(encoder, REG_REFLINE_MSB, ref_line);
-        reg_write16(encoder, REG_NPIX_MSB, n_pix);
-        reg_write16(encoder, REG_NLINE_MSB, n_line);
-        reg_write16(encoder, REG_VS_LINE_STRT_1_MSB, vs1_line_s);
-        reg_write16(encoder, REG_VS_PIX_STRT_1_MSB, vs1_pix_s);
-        reg_write16(encoder, REG_VS_LINE_END_1_MSB, vs1_line_e);
-        reg_write16(encoder, REG_VS_PIX_END_1_MSB, vs1_pix_e);
-        reg_write16(encoder, REG_VS_LINE_STRT_2_MSB, vs2_line_s);
-        reg_write16(encoder, REG_VS_PIX_STRT_2_MSB, vs2_pix_s);
-        reg_write16(encoder, REG_VS_LINE_END_2_MSB, vs2_line_e);
-        reg_write16(encoder, REG_VS_PIX_END_2_MSB, vs2_pix_e);
-        reg_write16(encoder, REG_HS_PIX_START_MSB, hs_pix_s);
-        reg_write16(encoder, REG_HS_PIX_STOP_MSB, hs_pix_e);
-        reg_write16(encoder, REG_VWIN_START_1_MSB, vwin1_line_s);
-        reg_write16(encoder, REG_VWIN_END_1_MSB, vwin1_line_e);
-        reg_write16(encoder, REG_VWIN_START_2_MSB, vwin2_line_s);
-        reg_write16(encoder, REG_VWIN_END_2_MSB, vwin2_line_e);
-        reg_write16(encoder, REG_DE_START_MSB, de_pix_s);
-        reg_write16(encoder, REG_DE_STOP_MSB, de_pix_e);
-
-        if (priv->rev == TDA19988) {
-                /* let incoming pixels fill the active space (if any) */
-                reg_write(encoder, REG_ENABLE_SPACE, 0x00);
-        }
+        reg_write(priv, REG_TBG_CNTRL_1, reg);
 
         /* must be last register set: */
-        reg_clear(encoder, REG_TBG_CNTRL_0, TBG_CNTRL_0_SYNC_ONCE);
+        reg_write(priv, REG_TBG_CNTRL_0, 0);
 
         /* Only setup the info frames if the sink is HDMI */
         if (priv->is_hdmi_sink) {
                 /* We need to turn HDMI HDCP stuff on to get audio through */
-                reg_clear(encoder, REG_TBG_CNTRL_1, TBG_CNTRL_1_DWIN_DIS);
-                reg_write(encoder, REG_ENC_CNTRL, ENC_CNTRL_CTL_CODE(1));
-                reg_set(encoder, REG_TX33, TX33_HDMI);
+                reg &= ~TBG_CNTRL_1_DWIN_DIS;
+                reg_write(priv, REG_TBG_CNTRL_1, reg);
+                reg_write(priv, REG_ENC_CNTRL, ENC_CNTRL_CTL_CODE(1));
+                reg_set(priv, REG_TX33, TX33_HDMI);
 
-                tda998x_write_avi(encoder, adjusted_mode);
+                tda998x_write_avi(priv, adjusted_mode);
 
                 if (priv->params.audio_cfg)
-                        tda998x_configure_audio(encoder, adjusted_mode,
+                        tda998x_configure_audio(priv, adjusted_mode,
                                                 &priv->params);
         }
 }
@@ -943,7 +1009,9 @@ static enum drm_connector_status
 tda998x_encoder_detect(struct drm_encoder *encoder,
                       struct drm_connector *connector)
 {
-        uint8_t val = cec_read(encoder, REG_CEC_RXSHPDLEV);
+        struct tda998x_priv *priv = to_tda998x_priv(encoder);
+        uint8_t val = cec_read(priv, REG_CEC_RXSHPDLEV);
+
         return (val & CEC_RXSHPDLEV_HPD) ? connector_status_connected :
                         connector_status_disconnected;
 }
@@ -951,46 +1019,57 @@ tda998x_encoder_detect(struct drm_encoder *encoder,
 static int
 read_edid_block(struct drm_encoder *encoder, uint8_t *buf, int blk)
 {
+        struct tda998x_priv *priv = to_tda998x_priv(encoder);
         uint8_t offset, segptr;
         int ret, i;
 
-        /* enable EDID read irq: */
-        reg_set(encoder, REG_INT_FLAGS_2, INT_FLAGS_2_EDID_BLK_RD);
-
         offset = (blk & 1) ? 128 : 0;
         segptr = blk / 2;
 
-        reg_write(encoder, REG_DDC_ADDR, 0xa0);
-        reg_write(encoder, REG_DDC_OFFS, offset);
-        reg_write(encoder, REG_DDC_SEGM_ADDR, 0x60);
-        reg_write(encoder, REG_DDC_SEGM, segptr);
+        reg_write(priv, REG_DDC_ADDR, 0xa0);
+        reg_write(priv, REG_DDC_OFFS, offset);
+        reg_write(priv, REG_DDC_SEGM_ADDR, 0x60);
+        reg_write(priv, REG_DDC_SEGM, segptr);
 
         /* enable reading EDID: */
-        reg_write(encoder, REG_EDID_CTRL, 0x1);
+        priv->wq_edid_wait = 1;
+        reg_write(priv, REG_EDID_CTRL, 0x1);
 
         /* flag must be cleared by sw: */
-        reg_write(encoder, REG_EDID_CTRL, 0x0);
+        reg_write(priv, REG_EDID_CTRL, 0x0);
 
         /* wait for block read to complete: */
-        for (i = 100; i > 0; i--) {
-                uint8_t val = reg_read(encoder, REG_INT_FLAGS_2);
-                if (val & INT_FLAGS_2_EDID_BLK_RD)
-                        break;
-                msleep(1);
+        if (priv->hdmi->irq) {
+                i = wait_event_timeout(priv->wq_edid,
+                                        !priv->wq_edid_wait,
+                                        msecs_to_jiffies(100));
+                if (i < 0) {
+                        dev_err(&priv->hdmi->dev, "read edid wait err %d\n", i);
+                        return i;
+                }
+        } else {
+                for (i = 10; i > 0; i--) {
+                        msleep(10);
+                        ret = reg_read(priv, REG_INT_FLAGS_2);
+                        if (ret < 0)
+                                return ret;
+                        if (ret & INT_FLAGS_2_EDID_BLK_RD)
+                                break;
+                }
         }
 
-        if (i == 0)
+        if (i == 0) {
+                dev_err(&priv->hdmi->dev, "read edid timeout\n");
                 return -ETIMEDOUT;
+        }
 
-        ret = reg_read_range(encoder, REG_EDID_DATA_0, buf, EDID_LENGTH);
+        ret = reg_read_range(priv, REG_EDID_DATA_0, buf, EDID_LENGTH);
         if (ret != EDID_LENGTH) {
-                dev_err(encoder->dev->dev, "failed to read edid block %d: %d",
-                                blk, ret);
+                dev_err(&priv->hdmi->dev, "failed to read edid block %d: %d\n",
+                        blk, ret);
                 return ret;
         }
 
-        reg_clear(encoder, REG_INT_FLAGS_2, INT_FLAGS_2_EDID_BLK_RD);
-
         return 0;
 }
 
@@ -998,7 +1077,7 @@ static uint8_t *
 do_get_edid(struct drm_encoder *encoder)
 {
         struct tda998x_priv *priv = to_tda998x_priv(encoder);
-        int j = 0, valid_extensions = 0;
+        int j, valid_extensions = 0;
         uint8_t *block, *new;
         bool print_bad_edid = drm_debug & DRM_UT_KMS;
 
@@ -1006,7 +1085,7 @@ do_get_edid(struct drm_encoder *encoder)
                 return NULL;
 
         if (priv->rev == TDA19988)
-                reg_clear(encoder, REG_TX4, TX4_PD_RAM);
+                reg_clear(priv, REG_TX4, TX4_PD_RAM);
 
         /* base block fetch */
         if (read_edid_block(encoder, block, 0))
@@ -1046,14 +1125,14 @@ do_get_edid(struct drm_encoder *encoder)
 
 done:
         if (priv->rev == TDA19988)
-                reg_set(encoder, REG_TX4, TX4_PD_RAM);
+                reg_set(priv, REG_TX4, TX4_PD_RAM);
 
         return block;
 
 fail:
         if (priv->rev == TDA19988)
-                reg_set(encoder, REG_TX4, TX4_PD_RAM);
-        dev_warn(encoder->dev->dev, "failed to read EDID\n");
+                reg_set(priv, REG_TX4, TX4_PD_RAM);
+        dev_warn(&priv->hdmi->dev, "failed to read EDID\n");
         kfree(block);
         return NULL;
 }
@@ -1080,7 +1159,13 @@ static int
 tda998x_encoder_create_resources(struct drm_encoder *encoder,
                                 struct drm_connector *connector)
 {
-        DBG("");
+        struct tda998x_priv *priv = to_tda998x_priv(encoder);
+
+        if (priv->hdmi->irq)
+                connector->polled = DRM_CONNECTOR_POLL_HPD;
+        else
+                connector->polled = DRM_CONNECTOR_POLL_CONNECT |
+                        DRM_CONNECTOR_POLL_DISCONNECT;
         return 0;
 }
 
@@ -1099,6 +1184,13 @@ tda998x_encoder_destroy(struct drm_encoder *encoder)
 {
         struct tda998x_priv *priv = to_tda998x_priv(encoder);
         drm_i2c_encoder_destroy(encoder);
+
+        /* disable all IRQs and free the IRQ handler */
+        cec_write(priv, REG_CEC_RXSHPDINTENA, 0);
+        reg_clear(priv, REG_INT_FLAGS_2, INT_FLAGS_2_EDID_BLK_RD);
+        if (priv->hdmi->irq)
+                free_irq(priv->hdmi->irq, priv);
+
         if (priv->cec)
                 i2c_unregister_device(priv->cec);
         kfree(priv);
@@ -1138,8 +1230,10 @@ tda998x_encoder_init(struct i2c_client *client,
                     struct drm_device *dev,
                     struct drm_encoder_slave *encoder_slave)
 {
-        struct drm_encoder *encoder = &encoder_slave->base;
         struct tda998x_priv *priv;
+        struct device_node *np = client->dev.of_node;
+        u32 video;
+        int rev_lo, rev_hi, ret;
 
         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
         if (!priv)
@@ -1150,52 +1244,113 @@ tda998x_encoder_init(struct i2c_client *client,
         priv->vip_cntrl_2 = VIP_CNTRL_2_SWAP_E(4) | VIP_CNTRL_2_SWAP_F(5);
 
         priv->current_page = 0xff;
+        priv->hdmi = client;
         priv->cec = i2c_new_dummy(client->adapter, 0x34);
         if (!priv->cec) {
                 kfree(priv);
                 return -ENODEV;
         }
+
+        priv->encoder = &encoder_slave->base;
         priv->dpms = DRM_MODE_DPMS_OFF;
 
         encoder_slave->slave_priv = priv;
         encoder_slave->slave_funcs = &tda998x_encoder_funcs;
 
         /* wake up the device: */
-        cec_write(encoder, REG_CEC_ENAMODS,
+        cec_write(priv, REG_CEC_ENAMODS,
                         CEC_ENAMODS_EN_RXSENS | CEC_ENAMODS_EN_HDMI);
 
-        tda998x_reset(encoder);
+        tda998x_reset(priv);
 
         /* read version: */
-        priv->rev = reg_read(encoder, REG_VERSION_LSB) |
-                        reg_read(encoder, REG_VERSION_MSB) << 8;
+        rev_lo = reg_read(priv, REG_VERSION_LSB);
+        rev_hi = reg_read(priv, REG_VERSION_MSB);
+        if (rev_lo < 0 || rev_hi < 0) {
+                ret = rev_lo < 0 ? rev_lo : rev_hi;
+                goto fail;
+        }
+
+        priv->rev = rev_lo | rev_hi << 8;
 
         /* mask off feature bits: */
         priv->rev &= ~0x30; /* not-hdcp and not-scalar bit */
 
         switch (priv->rev) {
-        case TDA9989N2:  dev_info(dev->dev, "found TDA9989 n2");  break;
-        case TDA19989:   dev_info(dev->dev, "found TDA19989");    break;
-        case TDA19989N2: dev_info(dev->dev, "found TDA19989 n2"); break;
-        case TDA19988:   dev_info(dev->dev, "found TDA19988");    break;
+        case TDA9989N2:
+                dev_info(&client->dev, "found TDA9989 n2");
+                break;
+        case TDA19989:
+                dev_info(&client->dev, "found TDA19989");
+                break;
+        case TDA19989N2:
+                dev_info(&client->dev, "found TDA19989 n2");
+                break;
+        case TDA19988:
+                dev_info(&client->dev, "found TDA19988");
+                break;
         default:
-                DBG("found unsupported device: %04x", priv->rev);
+                dev_err(&client->dev, "found unsupported device: %04x\n",
+                        priv->rev);
                 goto fail;
         }
 
         /* after reset, enable DDC: */
-        reg_write(encoder, REG_DDC_DISABLE, 0x00);
+        reg_write(priv, REG_DDC_DISABLE, 0x00);
 
         /* set clock on DDC channel: */
-        reg_write(encoder, REG_TX3, 39);
+        reg_write(priv, REG_TX3, 39);
 
         /* if necessary, disable multi-master: */
         if (priv->rev == TDA19989)
-                reg_set(encoder, REG_I2C_MASTER, I2C_MASTER_DIS_MM);
+                reg_set(priv, REG_I2C_MASTER, I2C_MASTER_DIS_MM);
 
-        cec_write(encoder, REG_CEC_FRO_IM_CLK_CTRL,
+        cec_write(priv, REG_CEC_FRO_IM_CLK_CTRL,
                         CEC_FRO_IM_CLK_CTRL_GHOST_DIS | CEC_FRO_IM_CLK_CTRL_IMCLK_SEL);
 
+        /* initialize the optional IRQ */
+        if (client->irq) {
+                int irqf_trigger;
+
+                /* init read EDID waitqueue */
+                init_waitqueue_head(&priv->wq_edid);
+
+                /* clear pending interrupts */
+                reg_read(priv, REG_INT_FLAGS_0);
+                reg_read(priv, REG_INT_FLAGS_1);
+                reg_read(priv, REG_INT_FLAGS_2);
+
+                irqf_trigger =
+                        irqd_get_trigger_type(irq_get_irq_data(client->irq));
+                ret = request_threaded_irq(client->irq, NULL,
+                                           tda998x_irq_thread,
+                                           irqf_trigger | IRQF_ONESHOT,
+                                           "tda998x", priv);
+                if (ret) {
+                        dev_err(&client->dev,
+                                "failed to request IRQ#%u: %d\n",
+                                client->irq, ret);
+                        goto fail;
+                }
+
+                /* enable HPD irq */
+                cec_write(priv, REG_CEC_RXSHPDINTENA, CEC_RXSHPDLEV_HPD);
+        }
+
+        /* enable EDID read irq: */
+        reg_set(priv, REG_INT_FLAGS_2, INT_FLAGS_2_EDID_BLK_RD);
+
+        if (!np)
+                return 0;               /* non-DT */
+
+        /* get the optional video properties */
+        ret = of_property_read_u32(np, "video-ports", &video);
+        if (ret == 0) {
+                priv->vip_cntrl_0 = video >> 16;
+                priv->vip_cntrl_1 = video >> 8;
+                priv->vip_cntrl_2 = video;
+        }
+
         return 0;
 
 fail:
@@ -1210,6 +1365,14 @@ fail:
         return -ENXIO;
 }
 
+#ifdef CONFIG_OF
+static const struct of_device_id tda998x_dt_ids[] = {
+        { .compatible = "nxp,tda998x", },
+        { }
+};
+MODULE_DEVICE_TABLE(of, tda998x_dt_ids);
+#endif
+
 static struct i2c_device_id tda998x_ids[] = {
         { "tda998x", 0 },
         { }
@@ -1222,6 +1385,7 @@ static struct drm_i2c_encoder_driver tda998x_driver = {
                 .remove = tda998x_remove,
                 .driver = {
                         .name = "tda998x",
+                        .of_match_table = of_match_ptr(tda998x_dt_ids),
                 },
                 .id_table = tda998x_ids,
         },
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 9fd44f5f3b3b..b1445b73465b 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -3,57 +3,69 @@
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
 ccflags-y := -Iinclude/drm
-i915-y := i915_drv.o i915_dma.o i915_irq.o \
-          i915_gpu_error.o \
+
+# Please keep these build lists sorted!
+
+# core driver code
+i915-y := i915_drv.o \
+          i915_params.o \
           i915_suspend.o \
-          i915_gem.o \
+          i915_sysfs.o \
+          intel_pm.o
+i915-$(CONFIG_COMPAT)   += i915_ioc32.o
+i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o
+
+# GEM code
+i915-y += i915_cmd_parser.o \
           i915_gem_context.o \
           i915_gem_debug.o \
+          i915_gem_dmabuf.o \
           i915_gem_evict.o \
           i915_gem_execbuffer.o \
           i915_gem_gtt.o \
+          i915_gem.o \
           i915_gem_stolen.o \
           i915_gem_tiling.o \
-          i915_sysfs.o \
+          i915_gpu_error.o \
+          i915_irq.o \
           i915_trace_points.o \
-          i915_ums.o \
+          intel_ringbuffer.o \
+          intel_uncore.o
+
+# modesetting core code
+i915-y += intel_bios.o \
           intel_display.o \
-          intel_crt.o \
-          intel_lvds.o \
-          intel_dsi.o \
-          intel_dsi_cmd.o \
-          intel_dsi_pll.o \
-          intel_bios.o \
-          intel_ddi.o \
-          intel_dp.o \
-          intel_hdmi.o \
-          intel_sdvo.o \
           intel_modes.o \
-          intel_panel.o \
-          intel_pm.o \
-          intel_i2c.o \
-          intel_tv.o \
-          intel_dvo.o \
-          intel_ringbuffer.o \
           intel_overlay.o \
-          intel_sprite.o \
           intel_sideband.o \
-          intel_uncore.o \
+          intel_sprite.o
+i915-$(CONFIG_ACPI)             += intel_acpi.o intel_opregion.o
+i915-$(CONFIG_DRM_I915_FBDEV)   += intel_fbdev.o
+
+# modesetting output/encoder code
+i915-y += dvo_ch7017.o \
           dvo_ch7xxx.o \
-          dvo_ch7017.o \
           dvo_ivch.o \
-          dvo_tfp410.o \
-          dvo_sil164.o \
           dvo_ns2501.o \
-          i915_gem_dmabuf.o
-
-i915-$(CONFIG_COMPAT)   += i915_ioc32.o
-
-i915-$(CONFIG_ACPI)     += intel_acpi.o intel_opregion.o
-
-i915-$(CONFIG_DRM_I915_FBDEV) += intel_fbdev.o
+          dvo_sil164.o \
+          dvo_tfp410.o \
+          intel_crt.o \
+          intel_ddi.o \
+          intel_dp.o \
+          intel_dsi_cmd.o \
+          intel_dsi.o \
+          intel_dsi_pll.o \
+          intel_dvo.o \
+          intel_hdmi.o \
+          intel_i2c.o \
+          intel_lvds.o \
+          intel_panel.o \
+          intel_sdvo.o \
+          intel_tv.o
 
-i915-$(CONFIG_DEBUG_FS) += i915_debugfs.o
+# legacy horrors
+i915-y += i915_dma.o \
+          i915_ums.o
 
 obj-$(CONFIG_DRM_I915)  += i915.o
 
diff --git a/drivers/gpu/drm/i915/dvo_ch7xxx.c b/drivers/gpu/drm/i915/dvo_ch7xxx.c
index af42e94f6846..a0f5bdd69491 100644
--- a/drivers/gpu/drm/i915/dvo_ch7xxx.c
+++ b/drivers/gpu/drm/i915/dvo_ch7xxx.c
@@ -340,9 +340,9 @@ static void ch7xxx_dump_regs(struct intel_dvo_device *dvo)
         for (i = 0; i < CH7xxx_NUM_REGS; i++) {
                 uint8_t val;
                 if ((i % 8) == 0)
-                        DRM_LOG_KMS("\n %02X: ", i);
+                        DRM_DEBUG_KMS("\n %02X: ", i);
                 ch7xxx_readb(dvo, i, &val);
-                DRM_LOG_KMS("%02X ", val);
+                DRM_DEBUG_KMS("%02X ", val);
         }
 }
 
diff --git a/drivers/gpu/drm/i915/dvo_ivch.c b/drivers/gpu/drm/i915/dvo_ivch.c
index baaf65bf0bdd..0f1865d7d4d8 100644
--- a/drivers/gpu/drm/i915/dvo_ivch.c
+++ b/drivers/gpu/drm/i915/dvo_ivch.c
@@ -377,41 +377,41 @@ static void ivch_dump_regs(struct intel_dvo_device *dvo)
         uint16_t val;
 
         ivch_read(dvo, VR00, &val);
-        DRM_LOG_KMS("VR00: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR00: 0x%04x\n", val);
         ivch_read(dvo, VR01, &val);
-        DRM_LOG_KMS("VR01: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR01: 0x%04x\n", val);
         ivch_read(dvo, VR30, &val);
-        DRM_LOG_KMS("VR30: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR30: 0x%04x\n", val);
         ivch_read(dvo, VR40, &val);
-        DRM_LOG_KMS("VR40: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR40: 0x%04x\n", val);
 
         /* GPIO registers */
         ivch_read(dvo, VR80, &val);
-        DRM_LOG_KMS("VR80: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR80: 0x%04x\n", val);
         ivch_read(dvo, VR81, &val);
-        DRM_LOG_KMS("VR81: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR81: 0x%04x\n", val);
         ivch_read(dvo, VR82, &val);
-        DRM_LOG_KMS("VR82: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR82: 0x%04x\n", val);
         ivch_read(dvo, VR83, &val);
-        DRM_LOG_KMS("VR83: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR83: 0x%04x\n", val);
         ivch_read(dvo, VR84, &val);
-        DRM_LOG_KMS("VR84: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR84: 0x%04x\n", val);
         ivch_read(dvo, VR85, &val);
-        DRM_LOG_KMS("VR85: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR85: 0x%04x\n", val);
         ivch_read(dvo, VR86, &val);
-        DRM_LOG_KMS("VR86: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR86: 0x%04x\n", val);
         ivch_read(dvo, VR87, &val);
-        DRM_LOG_KMS("VR87: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR87: 0x%04x\n", val);
         ivch_read(dvo, VR88, &val);
-        DRM_LOG_KMS("VR88: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR88: 0x%04x\n", val);
 
         /* Scratch register 0 - AIM Panel type */
         ivch_read(dvo, VR8E, &val);
-        DRM_LOG_KMS("VR8E: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR8E: 0x%04x\n", val);
 
         /* Scratch register 1 - Status register */
         ivch_read(dvo, VR8F, &val);
-        DRM_LOG_KMS("VR8F: 0x%04x\n", val);
+        DRM_DEBUG_KMS("VR8F: 0x%04x\n", val);
 }
 
 static void ivch_destroy(struct intel_dvo_device *dvo)
diff --git a/drivers/gpu/drm/i915/dvo_ns2501.c b/drivers/gpu/drm/i915/dvo_ns2501.c
index 954acb2c7021..8155ded79079 100644
--- a/drivers/gpu/drm/i915/dvo_ns2501.c
+++ b/drivers/gpu/drm/i915/dvo_ns2501.c
@@ -490,15 +490,15 @@ static void ns2501_dump_regs(struct intel_dvo_device *dvo)
         uint8_t val;
 
         ns2501_readb(dvo, NS2501_FREQ_LO, &val);
-        DRM_LOG_KMS("NS2501_FREQ_LO: 0x%02x\n", val);
+        DRM_DEBUG_KMS("NS2501_FREQ_LO: 0x%02x\n", val);
         ns2501_readb(dvo, NS2501_FREQ_HI, &val);
-        DRM_LOG_KMS("NS2501_FREQ_HI: 0x%02x\n", val);
+        DRM_DEBUG_KMS("NS2501_FREQ_HI: 0x%02x\n", val);
         ns2501_readb(dvo, NS2501_REG8, &val);
-        DRM_LOG_KMS("NS2501_REG8: 0x%02x\n", val);
+        DRM_DEBUG_KMS("NS2501_REG8: 0x%02x\n", val);
         ns2501_readb(dvo, NS2501_REG9, &val);
-        DRM_LOG_KMS("NS2501_REG9: 0x%02x\n", val);
+        DRM_DEBUG_KMS("NS2501_REG9: 0x%02x\n", val);
         ns2501_readb(dvo, NS2501_REGC, &val);
-        DRM_LOG_KMS("NS2501_REGC: 0x%02x\n", val);
+        DRM_DEBUG_KMS("NS2501_REGC: 0x%02x\n", val);
 }
 
 static void ns2501_destroy(struct intel_dvo_device *dvo)
diff --git a/drivers/gpu/drm/i915/dvo_sil164.c b/drivers/gpu/drm/i915/dvo_sil164.c
index 4debd32e3e4c..7b3e9e936200 100644
--- a/drivers/gpu/drm/i915/dvo_sil164.c
+++ b/drivers/gpu/drm/i915/dvo_sil164.c
@@ -246,15 +246,15 @@ static void sil164_dump_regs(struct intel_dvo_device *dvo)
         uint8_t val;
 
         sil164_readb(dvo, SIL164_FREQ_LO, &val);
-        DRM_LOG_KMS("SIL164_FREQ_LO: 0x%02x\n", val);
+        DRM_DEBUG_KMS("SIL164_FREQ_LO: 0x%02x\n", val);
         sil164_readb(dvo, SIL164_FREQ_HI, &val);
-        DRM_LOG_KMS("SIL164_FREQ_HI: 0x%02x\n", val);
+        DRM_DEBUG_KMS("SIL164_FREQ_HI: 0x%02x\n", val);
         sil164_readb(dvo, SIL164_REG8, &val);
-        DRM_LOG_KMS("SIL164_REG8: 0x%02x\n", val);
+        DRM_DEBUG_KMS("SIL164_REG8: 0x%02x\n", val);
         sil164_readb(dvo, SIL164_REG9, &val);
-        DRM_LOG_KMS("SIL164_REG9: 0x%02x\n", val);
+        DRM_DEBUG_KMS("SIL164_REG9: 0x%02x\n", val);
         sil164_readb(dvo, SIL164_REGC, &val);
-        DRM_LOG_KMS("SIL164_REGC: 0x%02x\n", val);
+        DRM_DEBUG_KMS("SIL164_REGC: 0x%02x\n", val);
 }
 
 static void sil164_destroy(struct intel_dvo_device *dvo)
diff --git a/drivers/gpu/drm/i915/dvo_tfp410.c b/drivers/gpu/drm/i915/dvo_tfp410.c
index e17f1b07e915..12ea4b164692 100644
--- a/drivers/gpu/drm/i915/dvo_tfp410.c
+++ b/drivers/gpu/drm/i915/dvo_tfp410.c
@@ -267,33 +267,33 @@ static void tfp410_dump_regs(struct intel_dvo_device *dvo)
         uint8_t val, val2;
 
         tfp410_readb(dvo, TFP410_REV, &val);
-        DRM_LOG_KMS("TFP410_REV: 0x%02X\n", val);
+        DRM_DEBUG_KMS("TFP410_REV: 0x%02X\n", val);
         tfp410_readb(dvo, TFP410_CTL_1, &val);
-        DRM_LOG_KMS("TFP410_CTL1: 0x%02X\n", val);
+        DRM_DEBUG_KMS("TFP410_CTL1: 0x%02X\n", val);
         tfp410_readb(dvo, TFP410_CTL_2, &val);
-        DRM_LOG_KMS("TFP410_CTL2: 0x%02X\n", val);
+        DRM_DEBUG_KMS("TFP410_CTL2: 0x%02X\n", val);
         tfp410_readb(dvo, TFP410_CTL_3, &val);
-        DRM_LOG_KMS("TFP410_CTL3: 0x%02X\n", val);
+        DRM_DEBUG_KMS("TFP410_CTL3: 0x%02X\n", val);
         tfp410_readb(dvo, TFP410_USERCFG, &val);
-        DRM_LOG_KMS("TFP410_USERCFG: 0x%02X\n", val);
+        DRM_DEBUG_KMS("TFP410_USERCFG: 0x%02X\n", val);
         tfp410_readb(dvo, TFP410_DE_DLY, &val);
-        DRM_LOG_KMS("TFP410_DE_DLY: 0x%02X\n", val);
+        DRM_DEBUG_KMS("TFP410_DE_DLY: 0x%02X\n", val);
         tfp410_readb(dvo, TFP410_DE_CTL, &val);
-        DRM_LOG_KMS("TFP410_DE_CTL: 0x%02X\n", val);
+        DRM_DEBUG_KMS("TFP410_DE_CTL: 0x%02X\n", val);
         tfp410_readb(dvo, TFP410_DE_TOP, &val);
-        DRM_LOG_KMS("TFP410_DE_TOP: 0x%02X\n", val);
+        DRM_DEBUG_KMS("TFP410_DE_TOP: 0x%02X\n", val);
         tfp410_readb(dvo, TFP410_DE_CNT_LO, &val);
         tfp410_readb(dvo, TFP410_DE_CNT_HI, &val2);
-        DRM_LOG_KMS("TFP410_DE_CNT: 0x%02X%02X\n", val2, val);
+        DRM_DEBUG_KMS("TFP410_DE_CNT: 0x%02X%02X\n", val2, val);
         tfp410_readb(dvo, TFP410_DE_LIN_LO, &val);
         tfp410_readb(dvo, TFP410_DE_LIN_HI, &val2);
-        DRM_LOG_KMS("TFP410_DE_LIN: 0x%02X%02X\n", val2, val);
+        DRM_DEBUG_KMS("TFP410_DE_LIN: 0x%02X%02X\n", val2, val);
         tfp410_readb(dvo, TFP410_H_RES_LO, &val);
         tfp410_readb(dvo, TFP410_H_RES_HI, &val2);
-        DRM_LOG_KMS("TFP410_H_RES: 0x%02X%02X\n", val2, val);
+        DRM_DEBUG_KMS("TFP410_H_RES: 0x%02X%02X\n", val2, val);
         tfp410_readb(dvo, TFP410_V_RES_LO, &val);
         tfp410_readb(dvo, TFP410_V_RES_HI, &val2);
-        DRM_LOG_KMS("TFP410_V_RES: 0x%02X%02X\n", val2, val);
+        DRM_DEBUG_KMS("TFP410_V_RES: 0x%02X%02X\n", val2, val);
 }
 
 static void tfp410_destroy(struct intel_dvo_device *dvo)
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
new file mode 100644
index 000000000000..4cf6d020d513
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -0,0 +1,485 @@
+/*
+ * Copyright © 2013 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Brad Volkin <bradley.d.volkin@intel.com>
+ *
+ */
+
+#include "i915_drv.h"
+
+/**
+ * DOC: i915 batch buffer command parser
+ *
+ * Motivation:
+ * Certain OpenGL features (e.g. transform feedback, performance monitoring)
+ * require userspace code to submit batches containing commands such as
+ * MI_LOAD_REGISTER_IMM to access various registers. Unfortunately, some
+ * generations of the hardware will noop these commands in "unsecure" batches
+ * (which includes all userspace batches submitted via i915) even though the
+ * commands may be safe and represent the intended programming model of the
+ * device.
+ *
+ * The software command parser is similar in operation to the command parsing
+ * done in hardware for unsecure batches. However, the software parser allows
+ * some operations that would be noop'd by hardware, if the parser determines
+ * the operation is safe, and submits the batch as "secure" to prevent hardware
+ * parsing.
+ *
+ * Threats:
+ * At a high level, the hardware (and software) checks attempt to prevent
+ * granting userspace undue privileges. There are three categories of privilege.
+ *
+ * First, commands which are explicitly defined as privileged or which should
+ * only be used by the kernel driver. The parser generally rejects such
+ * commands, though it may allow some from the drm master process.
+ *
+ * Second, commands which access registers. To support correct/enhanced
+ * userspace functionality, particularly certain OpenGL extensions, the parser
+ * provides a whitelist of registers which userspace may safely access (for both
+ * normal and drm master processes).
+ *
+ * Third, commands which access privileged memory (i.e. GGTT, HWS page, etc).
+ * The parser always rejects such commands.
+ *
+ * The majority of the problematic commands fall in the MI_* range, with only a
+ * few specific commands on each ring (e.g. PIPE_CONTROL and MI_FLUSH_DW).
+ *
+ * Implementation:
+ * Each ring maintains tables of commands and registers which the parser uses in
+ * scanning batch buffers submitted to that ring.
+ *
+ * Since the set of commands that the parser must check for is significantly
+ * smaller than the number of commands supported, the parser tables contain only
+ * those commands required by the parser. This generally works because command
+ * opcode ranges have standard command length encodings. So for commands that
+ * the parser does not need to check, it can easily skip them. This is
+ * implementated via a per-ring length decoding vfunc.
+ *
+ * Unfortunately, there are a number of commands that do not follow the standard
+ * length encoding for their opcode range, primarily amongst the MI_* commands.
+ * To handle this, the parser provides a way to define explicit "skip" entries
+ * in the per-ring command tables.
+ *
+ * Other command table entries map fairly directly to high level categories
+ * mentioned above: rejected, master-only, register whitelist. The parser
+ * implements a number of checks, including the privileged memory checks, via a
+ * general bitmasking mechanism.
+ */
+
+static u32 gen7_render_get_cmd_length_mask(u32 cmd_header)
+{
+        u32 client = (cmd_header & INSTR_CLIENT_MASK) >> INSTR_CLIENT_SHIFT;
+        u32 subclient =
+                (cmd_header & INSTR_SUBCLIENT_MASK) >> INSTR_SUBCLIENT_SHIFT;
+
+        if (client == INSTR_MI_CLIENT)
+                return 0x3F;
+        else if (client == INSTR_RC_CLIENT) {
+                if (subclient == INSTR_MEDIA_SUBCLIENT)
+                        return 0xFFFF;
+                else
+                        return 0xFF;
+        }
+
+        DRM_DEBUG_DRIVER("CMD: Abnormal rcs cmd length! 0x%08X\n", cmd_header);
+        return 0;
+}
+
+static u32 gen7_bsd_get_cmd_length_mask(u32 cmd_header)
+{
+        u32 client = (cmd_header & INSTR_CLIENT_MASK) >> INSTR_CLIENT_SHIFT;
+        u32 subclient =
+                (cmd_header & INSTR_SUBCLIENT_MASK) >> INSTR_SUBCLIENT_SHIFT;
+
+        if (client == INSTR_MI_CLIENT)
+                return 0x3F;
+        else if (client == INSTR_RC_CLIENT) {
+                if (subclient == INSTR_MEDIA_SUBCLIENT)
+                        return 0xFFF;
+                else
+                        return 0xFF;
+        }
+
+        DRM_DEBUG_DRIVER("CMD: Abnormal bsd cmd length! 0x%08X\n", cmd_header);
+        return 0;
+}
+
+static u32 gen7_blt_get_cmd_length_mask(u32 cmd_header)
+{
+        u32 client = (cmd_header & INSTR_CLIENT_MASK) >> INSTR_CLIENT_SHIFT;
+
+        if (client == INSTR_MI_CLIENT)
+                return 0x3F;
+        else if (client == INSTR_BC_CLIENT)
+                return 0xFF;
+
+        DRM_DEBUG_DRIVER("CMD: Abnormal blt cmd length! 0x%08X\n", cmd_header);
+        return 0;
+}
+
+static void validate_cmds_sorted(struct intel_ring_buffer *ring)
+{
+        int i;
+
+        if (!ring->cmd_tables || ring->cmd_table_count == 0)
+                return;
+
+        for (i = 0; i < ring->cmd_table_count; i++) {
+                const struct drm_i915_cmd_table *table = &ring->cmd_tables[i];
+                u32 previous = 0;
+                int j;
+
+                for (j = 0; j < table->count; j++) {
+                        const struct drm_i915_cmd_descriptor *desc =
+                                &table->table[i];
+                        u32 curr = desc->cmd.value & desc->cmd.mask;
+
+                        if (curr < previous)
+                                DRM_ERROR("CMD: table not sorted ring=%d table=%d entry=%d cmd=0x%08X prev=0x%08X\n",
+                                          ring->id, i, j, curr, previous);
+
+                        previous = curr;
+                }
+        }
+}
+
+static void check_sorted(int ring_id, const u32 *reg_table, int reg_count)
+{
+        int i;
+        u32 previous = 0;
+
+        for (i = 0; i < reg_count; i++) {
+                u32 curr = reg_table[i];
+
+                if (curr < previous)
+                        DRM_ERROR("CMD: table not sorted ring=%d entry=%d reg=0x%08X prev=0x%08X\n",
+                                  ring_id, i, curr, previous);
+
+                previous = curr;
+        }
+}
+
+static void validate_regs_sorted(struct intel_ring_buffer *ring)
+{
+        check_sorted(ring->id, ring->reg_table, ring->reg_count);
+        check_sorted(ring->id, ring->master_reg_table, ring->master_reg_count);
+}
+
+/**
+ * i915_cmd_parser_init_ring() - set cmd parser related fields for a ringbuffer
+ * @ring: the ringbuffer to initialize
+ *
+ * Optionally initializes fields related to batch buffer command parsing in the
+ * struct intel_ring_buffer based on whether the platform requires software
+ * command parsing.
+ */
+void i915_cmd_parser_init_ring(struct intel_ring_buffer *ring)
+{
+        if (!IS_GEN7(ring->dev))
+                return;
+
+        switch (ring->id) {
+        case RCS:
+                ring->get_cmd_length_mask = gen7_render_get_cmd_length_mask;
+                break;
+        case VCS:
+                ring->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
+                break;
+        case BCS:
+                ring->get_cmd_length_mask = gen7_blt_get_cmd_length_mask;
+                break;
+        case VECS:
+                /* VECS can use the same length_mask function as VCS */
+                ring->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
+                break;
+        default:
+                DRM_ERROR("CMD: cmd_parser_init with unknown ring: %d\n",
+                          ring->id);
+                BUG();
+        }
+
+        validate_cmds_sorted(ring);
+        validate_regs_sorted(ring);
+}
+
+static const struct drm_i915_cmd_descriptor*
+find_cmd_in_table(const struct drm_i915_cmd_table *table,
+                  u32 cmd_header)
+{
+        int i;
+
+        for (i = 0; i < table->count; i++) {
+                const struct drm_i915_cmd_descriptor *desc = &table->table[i];
+                u32 masked_cmd = desc->cmd.mask & cmd_header;
+                u32 masked_value = desc->cmd.value & desc->cmd.mask;
+
+                if (masked_cmd == masked_value)
+                        return desc;
+        }
+
+        return NULL;
+}
+
+/*
+ * Returns a pointer to a descriptor for the command specified by cmd_header.
+ *
+ * The caller must supply space for a default descriptor via the default_desc
+ * parameter. If no descriptor for the specified command exists in the ring's
+ * command parser tables, this function fills in default_desc based on the
+ * ring's default length encoding and returns default_desc.
+ */
+static const struct drm_i915_cmd_descriptor*
+find_cmd(struct intel_ring_buffer *ring,
+         u32 cmd_header,
+         struct drm_i915_cmd_descriptor *default_desc)
+{
+        u32 mask;
+        int i;
+
+        for (i = 0; i < ring->cmd_table_count; i++) {
+                const struct drm_i915_cmd_descriptor *desc;
+
+                desc = find_cmd_in_table(&ring->cmd_tables[i], cmd_header);
+                if (desc)
+                        return desc;
+        }
+
+        mask = ring->get_cmd_length_mask(cmd_header);
+        if (!mask)
+                return NULL;
+
+        BUG_ON(!default_desc);
+        default_desc->flags = CMD_DESC_SKIP;
+        default_desc->length.mask = mask;
+
+        return default_desc;
+}
+
+static bool valid_reg(const u32 *table, int count, u32 addr)
+{
+        if (table && count != 0) {
+                int i;
+
+                for (i = 0; i < count; i++) {
+                        if (table[i] == addr)
+                                return true;
+                }
+        }
+
+        return false;
+}
+
+static u32 *vmap_batch(struct drm_i915_gem_object *obj)
+{
+        int i;
+        void *addr = NULL;
+        struct sg_page_iter sg_iter;
+        struct page **pages;
+
+        pages = drm_malloc_ab(obj->base.size >> PAGE_SHIFT, sizeof(*pages));
+        if (pages == NULL) {
+                DRM_DEBUG_DRIVER("Failed to get space for pages\n");
+                goto finish;
+        }
+
+        i = 0;
+        for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
+                pages[i] = sg_page_iter_page(&sg_iter);
+                i++;
+        }
+
+        addr = vmap(pages, i, 0, PAGE_KERNEL);
+        if (addr == NULL) {
+                DRM_DEBUG_DRIVER("Failed to vmap pages\n");
+                goto finish;
+        }
+
+finish:
+        if (pages)
+                drm_free_large(pages);
+        return (u32*)addr;
+}
+
+/**
+ * i915_needs_cmd_parser() - should a given ring use software command parsing?
+ * @ring: the ring in question
+ *
+ * Only certain platforms require software batch buffer command parsing, and
+ * only when enabled via module paramter.
+ *
+ * Return: true if the ring requires software command parsing
+ */
+bool i915_needs_cmd_parser(struct intel_ring_buffer *ring)
+{
+        /* No command tables indicates a platform without parsing */
+        if (!ring->cmd_tables)
+                return false;
+
+        return (i915.enable_cmd_parser == 1);
+}
+
+#define LENGTH_BIAS 2
+
+/**
+ * i915_parse_cmds() - parse a submitted batch buffer for privilege violations
+ * @ring: the ring on which the batch is to execute
+ * @batch_obj: the batch buffer in question
+ * @batch_start_offset: byte offset in the batch at which execution starts
+ * @is_master: is the submitting process the drm master?
+ *
+ * Parses the specified batch buffer looking for privilege violations as
+ * described in the overview.
+ *
+ * Return: non-zero if the parser finds violations or otherwise fails
+ */
+int i915_parse_cmds(struct intel_ring_buffer *ring,
+                    struct drm_i915_gem_object *batch_obj,
+                    u32 batch_start_offset,
+                    bool is_master)
+{
+        int ret = 0;
+        u32 *cmd, *batch_base, *batch_end;
+        struct drm_i915_cmd_descriptor default_desc = { 0 };
+        int needs_clflush = 0;
+
+        ret = i915_gem_obj_prepare_shmem_read(batch_obj, &needs_clflush);
+        if (ret) {
+                DRM_DEBUG_DRIVER("CMD: failed to prep read\n");
+                return ret;
+        }
+
+        batch_base = vmap_batch(batch_obj);
+        if (!batch_base) {
+                DRM_DEBUG_DRIVER("CMD: Failed to vmap batch\n");
+                i915_gem_object_unpin_pages(batch_obj);
+                return -ENOMEM;
+        }
+
+        if (needs_clflush)
+                drm_clflush_virt_range((char *)batch_base, batch_obj->base.size);
+
+        cmd = batch_base + (batch_start_offset / sizeof(*cmd));
+        batch_end = cmd + (batch_obj->base.size / sizeof(*batch_end));
+
+        while (cmd < batch_end) {
+                const struct drm_i915_cmd_descriptor *desc;
+                u32 length;
+
+                if (*cmd == MI_BATCH_BUFFER_END)
+                        break;
+
+                desc = find_cmd(ring, *cmd, &default_desc);
+                if (!desc) {
+                        DRM_DEBUG_DRIVER("CMD: Unrecognized command: 0x%08X\n",
+                                         *cmd);
+                        ret = -EINVAL;
+                        break;
+                }
+
+                if (desc->flags & CMD_DESC_FIXED)
+                        length = desc->length.fixed;
+                else
+                        length = ((*cmd & desc->length.mask) + LENGTH_BIAS);
+
+                if ((batch_end - cmd) < length) {
+                        DRM_DEBUG_DRIVER("CMD: Command length exceeds batch length: 0x%08X length=%d batchlen=%td\n",
+                                         *cmd,
+                                         length,
+                                         (unsigned long)(batch_end - cmd));
+                        ret = -EINVAL;
+                        break;
+                }
+
+                if (desc->flags & CMD_DESC_REJECT) {
+                        DRM_DEBUG_DRIVER("CMD: Rejected command: 0x%08X\n", *cmd);
+                        ret = -EINVAL;
+                        break;
+                }
+
+                if ((desc->flags & CMD_DESC_MASTER) && !is_master) {
+                        DRM_DEBUG_DRIVER("CMD: Rejected master-only command: 0x%08X\n",
+                                         *cmd);
+                        ret = -EINVAL;
+                        break;
+                }
+
+                if (desc->flags & CMD_DESC_REGISTER) {
+                        u32 reg_addr = cmd[desc->reg.offset] & desc->reg.mask;
+
+                        if (!valid_reg(ring->reg_table,
+                                       ring->reg_count, reg_addr)) {
+                                if (!is_master ||
+                                    !valid_reg(ring->master_reg_table,
+                                               ring->master_reg_count,
+                                               reg_addr)) {
+                                        DRM_DEBUG_DRIVER("CMD: Rejected register 0x%08X in command: 0x%08X (ring=%d)\n",
+                                                         reg_addr,
+                                                         *cmd,
+                                                         ring->id);
+                                        ret = -EINVAL;
+                                        break;
+                                }
+                        }
+                }
+
+                if (desc->flags & CMD_DESC_BITMASK) {
+                        int i;
+
+                        for (i = 0; i < MAX_CMD_DESC_BITMASKS; i++) {
+                                u32 dword;
+
+                                if (desc->bits[i].mask == 0)
+                                        break;
+
+                                dword = cmd[desc->bits[i].offset] &
+                                        desc->bits[i].mask;
+
+                                if (dword != desc->bits[i].expected) {
+                                        DRM_DEBUG_DRIVER("CMD: Rejected command 0x%08X for bitmask 0x%08X (exp=0x%08X act=0x%08X) (ring=%d)\n",
+                                                         *cmd,
+                                                         desc->bits[i].mask,
+                                                         desc->bits[i].expected,
+                                                         dword, ring->id);
+                                        ret = -EINVAL;
+                                        break;
+                                }
+                        }
+
+                        if (ret)
+                                break;
+                }
+
+                cmd += length;
+        }
+
+        if (cmd >= batch_end) {
+                DRM_DEBUG_DRIVER("CMD: Got to the end of the buffer w/o a BBE cmd!\n");
+                ret = -EINVAL;
+        }
+
+        vunmap(batch_base);
+
+        i915_gem_object_unpin_pages(batch_obj);
+
+        return ret;
+}
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index b2b46c52294c..195fe5bc0aac 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -98,7 +98,7 @@ static const char *get_pin_flag(struct drm_i915_gem_object *obj)
 {
         if (obj->user_pin_count > 0)
                 return "P";
-        else if (obj->pin_count > 0)
+        else if (i915_gem_obj_is_pinned(obj))
                 return "p";
         else
                 return " ";
@@ -123,6 +123,8 @@ static void
 describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
 {
         struct i915_vma *vma;
+        int pin_count = 0;
+
         seq_printf(m, "%pK: %s%s%s %8zdKiB %02x %02x %u %u %u%s%s%s",
                    &obj->base,
                    get_pin_flag(obj),
@@ -139,8 +141,10 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
                    obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
         if (obj->base.name)
                 seq_printf(m, " (name: %d)", obj->base.name);
-        if (obj->pin_count)
-                seq_printf(m, " (pinned x %d)", obj->pin_count);
+        list_for_each_entry(vma, &obj->vma_list, vma_link)
+                if (vma->pin_count > 0)
+                        pin_count++;
+                seq_printf(m, " (pinned x %d)", pin_count);
         if (obj->pin_display)
                 seq_printf(m, " (display)");
         if (obj->fence_reg != I915_FENCE_REG_NONE)
@@ -295,28 +299,62 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
 } while (0)
 
 struct file_stats {
+        struct drm_i915_file_private *file_priv;
         int count;
-        size_t total, active, inactive, unbound;
+        size_t total, unbound;
+        size_t global, shared;
+        size_t active, inactive;
 };
 
 static int per_file_stats(int id, void *ptr, void *data)
 {
         struct drm_i915_gem_object *obj = ptr;
         struct file_stats *stats = data;
+        struct i915_vma *vma;
 
         stats->count++;
         stats->total += obj->base.size;
 
-        if (i915_gem_obj_ggtt_bound(obj)) {
-                if (!list_empty(&obj->ring_list))
-                        stats->active += obj->base.size;
-                else
-                        stats->inactive += obj->base.size;
+        if (obj->base.name || obj->base.dma_buf)
+                stats->shared += obj->base.size;
+
+        if (USES_FULL_PPGTT(obj->base.dev)) {
+                list_for_each_entry(vma, &obj->vma_list, vma_link) {
+                        struct i915_hw_ppgtt *ppgtt;
+
+                        if (!drm_mm_node_allocated(&vma->node))
+                                continue;
+
+                        if (i915_is_ggtt(vma->vm)) {
+                                stats->global += obj->base.size;
+                                continue;
+                        }
+
+                        ppgtt = container_of(vma->vm, struct i915_hw_ppgtt, base);
+                        if (ppgtt->ctx && ppgtt->ctx->file_priv != stats->file_priv)
+                                continue;
+
+                        if (obj->ring) /* XXX per-vma statistic */
+                                stats->active += obj->base.size;
+                        else
+                                stats->inactive += obj->base.size;
+
+                        return 0;
+                }
         } else {
-                if (!list_empty(&obj->global_list))
-                        stats->unbound += obj->base.size;
+                if (i915_gem_obj_ggtt_bound(obj)) {
+                        stats->global += obj->base.size;
+                        if (obj->ring)
+                                stats->active += obj->base.size;
+                        else
+                                stats->inactive += obj->base.size;
+                        return 0;
+                }
         }
 
+        if (!list_empty(&obj->global_list))
+                stats->unbound += obj->base.size;
+
         return 0;
 }
 
@@ -407,6 +445,7 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
                 struct task_struct *task;
 
                 memset(&stats, 0, sizeof(stats));
+                stats.file_priv = file->driver_priv;
                 idr_for_each(&file->object_idr, per_file_stats, &stats);
                 /*
                  * Although we have a valid reference on file->pid, that does
@@ -416,12 +455,14 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
                  */
                 rcu_read_lock();
                 task = pid_task(file->pid, PIDTYPE_PID);
-                seq_printf(m, "%s: %u objects, %zu bytes (%zu active, %zu inactive, %zu unbound)\n",
+                seq_printf(m, "%s: %u objects, %zu bytes (%zu active, %zu inactive, %zu global, %zu shared, %zu unbound)\n",
                            task ? task->comm : "<unknown>",
                            stats.count,
                            stats.total,
                            stats.active,
                            stats.inactive,
+                           stats.global,
+                           stats.shared,
                            stats.unbound);
                 rcu_read_unlock();
         }
@@ -447,7 +488,7 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data)
 
         total_obj_size = total_gtt_size = count = 0;
         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
-                if (list == PINNED_LIST && obj->pin_count == 0)
+                if (list == PINNED_LIST && !i915_gem_obj_is_pinned(obj))
                         continue;
 
                 seq_puts(m, "   ");
@@ -520,7 +561,7 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring;
         struct drm_i915_gem_request *gem_request;
         int ret, count, i;
@@ -565,7 +606,7 @@ static int i915_gem_seqno_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring;
         int ret, i;
 
@@ -588,7 +629,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring;
         int ret, i, pipe;
 
@@ -598,7 +639,6 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
         intel_runtime_pm_get(dev_priv);
 
         if (INTEL_INFO(dev)->gen >= 8) {
-                int i;
                 seq_printf(m, "Master Interrupt Control:\t%08x\n",
                            I915_READ(GEN8_MASTER_IRQ));
 
@@ -611,16 +651,16 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
                                    i, I915_READ(GEN8_GT_IER(i)));
                 }
 
-                for_each_pipe(i) {
+                for_each_pipe(pipe) {
                         seq_printf(m, "Pipe %c IMR:\t%08x\n",
-                                   pipe_name(i),
-                                   I915_READ(GEN8_DE_PIPE_IMR(i)));
+                                   pipe_name(pipe),
+                                   I915_READ(GEN8_DE_PIPE_IMR(pipe)));
                         seq_printf(m, "Pipe %c IIR:\t%08x\n",
-                                   pipe_name(i),
-                                   I915_READ(GEN8_DE_PIPE_IIR(i)));
+                                   pipe_name(pipe),
+                                   I915_READ(GEN8_DE_PIPE_IIR(pipe)));
                         seq_printf(m, "Pipe %c IER:\t%08x\n",
-                                   pipe_name(i),
-                                   I915_READ(GEN8_DE_PIPE_IER(i)));
+                                   pipe_name(pipe),
+                                   I915_READ(GEN8_DE_PIPE_IER(pipe)));
                 }
 
                 seq_printf(m, "Display Engine port interrupt mask:\t%08x\n",
@@ -712,8 +752,6 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
                 seq_printf(m, "Graphics Interrupt mask:         %08x\n",
                            I915_READ(GTIMR));
         }
-        seq_printf(m, "Interrupts received: %d\n",
-                   atomic_read(&dev_priv->irq_received));
         for_each_ring(ring, dev_priv, i) {
                 if (INTEL_INFO(dev)->gen >= 6) {
                         seq_printf(m,
@@ -732,7 +770,7 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int i, ret;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -761,7 +799,7 @@ static int i915_hws_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring;
         const u32 *hws;
         int i;
@@ -872,7 +910,7 @@ static int
 i915_next_seqno_get(void *data, u64 *val)
 {
         struct drm_device *dev = data;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -909,7 +947,7 @@ static int i915_rstdby_delays(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u16 crstanddelay;
         int ret;
 
@@ -932,7 +970,7 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret = 0;
 
         intel_runtime_pm_get(dev_priv);
@@ -1025,7 +1063,7 @@ static int i915_cur_delayinfo(struct seq_file *m, void *unused)
                            max_freq * GT_FREQUENCY_MULTIPLIER);
 
                 seq_printf(m, "Max overclocked frequency: %dMHz\n",
-                           dev_priv->rps.hw_max * GT_FREQUENCY_MULTIPLIER);
+                           dev_priv->rps.max_freq * GT_FREQUENCY_MULTIPLIER);
         } else if (IS_VALLEYVIEW(dev)) {
                 u32 freq_sts, val;
 
@@ -1058,7 +1096,7 @@ static int i915_delayfreq_table(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 delayfreq;
         int ret, i;
 
@@ -1089,7 +1127,7 @@ static int i915_inttoext_table(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 inttoext;
         int ret, i;
 
@@ -1113,7 +1151,7 @@ static int ironlake_drpc_info(struct seq_file *m)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 rgvmodectl, rstdbyctl;
         u16 crstandvid;
         int ret;
@@ -1339,13 +1377,15 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         if (!HAS_FBC(dev)) {
                 seq_puts(m, "FBC unsupported on this chipset\n");
                 return 0;
         }
 
+        intel_runtime_pm_get(dev_priv);
+
         if (intel_fbc_enabled(dev)) {
                 seq_puts(m, "FBC enabled\n");
         } else {
@@ -1389,6 +1429,9 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
                 }
                 seq_putc(m, '\n');
         }
+
+        intel_runtime_pm_put(dev_priv);
+
         return 0;
 }
 
@@ -1403,11 +1446,15 @@ static int i915_ips_status(struct seq_file *m, void *unused)
                 return 0;
         }
 
+        intel_runtime_pm_get(dev_priv);
+
         if (IS_BROADWELL(dev) || I915_READ(IPS_CTL) & IPS_ENABLE)
                 seq_puts(m, "enabled\n");
         else
                 seq_puts(m, "disabled\n");
 
+        intel_runtime_pm_put(dev_priv);
+
         return 0;
 }
 
@@ -1415,9 +1462,11 @@ static int i915_sr_status(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         bool sr_enabled = false;
 
+        intel_runtime_pm_get(dev_priv);
+
         if (HAS_PCH_SPLIT(dev))
                 sr_enabled = I915_READ(WM1_LP_ILK) & WM1_LP_SR_EN;
         else if (IS_CRESTLINE(dev) || IS_I945G(dev) || IS_I945GM(dev))
@@ -1427,6 +1476,8 @@ static int i915_sr_status(struct seq_file *m, void *unused)
         else if (IS_PINEVIEW(dev))
                 sr_enabled = I915_READ(DSPFW3) & PINEVIEW_SELF_REFRESH_EN;
 
+        intel_runtime_pm_put(dev_priv);
+
         seq_printf(m, "self-refresh: %s\n",
                    sr_enabled ? "enabled" : "disabled");
 
@@ -1437,7 +1488,7 @@ static int i915_emon_status(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long temp, chipset, gfx;
         int ret;
 
@@ -1465,8 +1516,8 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        int ret;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret = 0;
         int gpu_freq, ia_freq;
 
         if (!(IS_GEN6(dev) || IS_GEN7(dev))) {
@@ -1474,17 +1525,18 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
                 return 0;
         }
 
+        intel_runtime_pm_get(dev_priv);
+
         flush_delayed_work(&dev_priv->rps.delayed_resume_work);
 
         ret = mutex_lock_interruptible(&dev_priv->rps.hw_lock);
         if (ret)
-                return ret;
-        intel_runtime_pm_get(dev_priv);
+                goto out;
 
         seq_puts(m, "GPU freq (MHz)\tEffective CPU freq (MHz)\tEffective Ring freq (MHz)\n");
 
-        for (gpu_freq = dev_priv->rps.min_delay;
-             gpu_freq <= dev_priv->rps.max_delay;
+        for (gpu_freq = dev_priv->rps.min_freq_softlimit;
+             gpu_freq <= dev_priv->rps.max_freq_softlimit;
              gpu_freq++) {
                 ia_freq = gpu_freq;
                 sandybridge_pcode_read(dev_priv,
@@ -1496,17 +1548,18 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
                            ((ia_freq >> 8) & 0xff) * 100);
         }
 
-        intel_runtime_pm_put(dev_priv);
         mutex_unlock(&dev_priv->rps.hw_lock);
 
-        return 0;
+out:
+        intel_runtime_pm_put(dev_priv);
+        return ret;
 }
 
 static int i915_gfxec(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -1526,7 +1579,7 @@ static int i915_opregion(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_opregion *opregion = &dev_priv->opregion;
         void *data = kmalloc(OPREGION_SIZE, GFP_KERNEL);
         int ret;
@@ -1600,7 +1653,7 @@ static int i915_context_status(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring;
         struct i915_hw_context *ctx;
         int ret, i;
@@ -1733,6 +1786,17 @@ static int i915_swizzle_info(struct seq_file *m, void *data)
         return 0;
 }
 
+static int per_file_ctx(int id, void *ptr, void *data)
+{
+        struct i915_hw_context *ctx = ptr;
+        struct seq_file *m = data;
+        struct i915_hw_ppgtt *ppgtt = ctx_to_ppgtt(ctx);
+
+        ppgtt->debug_dump(ppgtt, m);
+
+        return 0;
+}
+
 static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1744,7 +1808,7 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)
                 return;
 
         seq_printf(m, "Page directories: %d\n", ppgtt->num_pd_pages);
-        seq_printf(m, "Page tables: %d\n", ppgtt->num_pt_pages);
+        seq_printf(m, "Page tables: %d\n", ppgtt->num_pd_entries);
         for_each_ring(ring, dev_priv, unused) {
                 seq_printf(m, "%s\n", ring->name);
                 for (i = 0; i < 4; i++) {
@@ -1762,6 +1826,7 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring;
+        struct drm_file *file;
         int i;
 
         if (INTEL_INFO(dev)->gen == 6)
@@ -1780,6 +1845,20 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)
 
                 seq_puts(m, "aliasing PPGTT:\n");
                 seq_printf(m, "pd gtt offset: 0x%08x\n", ppgtt->pd_offset);
+
+                ppgtt->debug_dump(ppgtt, m);
+        } else
+                return;
+
+        list_for_each_entry_reverse(file, &dev->filelist, lhead) {
+                struct drm_i915_file_private *file_priv = file->driver_priv;
+                struct i915_hw_ppgtt *pvt_ppgtt;
+
+                pvt_ppgtt = ctx_to_ppgtt(file_priv->private_default_ctx);
+                seq_printf(m, "proc: %s\n",
+                           get_pid_task(file->pid, PIDTYPE_PID)->comm);
+                seq_puts(m, "  default context:\n");
+                idr_for_each(&file_priv->context_idr, per_file_ctx, m);
         }
         seq_printf(m, "ECOCHK: 0x%08x\n", I915_READ(GAM_ECOCHK));
 }
@@ -1892,6 +1971,47 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
         return 0;
 }
 
+static int i915_sink_crc(struct seq_file *m, void *data)
+{
+        struct drm_info_node *node = m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct intel_encoder *encoder;
+        struct intel_connector *connector;
+        struct intel_dp *intel_dp = NULL;
+        int ret;
+        u8 crc[6];
+
+        drm_modeset_lock_all(dev);
+        list_for_each_entry(connector, &dev->mode_config.connector_list,
+                            base.head) {
+
+                if (connector->base.dpms != DRM_MODE_DPMS_ON)
+                        continue;
+
+                if (!connector->base.encoder)
+                        continue;
+
+                encoder = to_intel_encoder(connector->base.encoder);
+                if (encoder->type != INTEL_OUTPUT_EDP)
+                        continue;
+
+                intel_dp = enc_to_intel_dp(&encoder->base);
+
+                ret = intel_dp_sink_crc(intel_dp, crc);
+                if (ret)
+                        goto out;
+
+                seq_printf(m, "%02x%02x%02x%02x%02x%02x\n",
+                           crc[0], crc[1], crc[2],
+                           crc[3], crc[4], crc[5]);
+                goto out;
+        }
+        ret = -ENODEV;
+out:
+        drm_modeset_unlock_all(dev);
+        return ret;
+}
+
 static int i915_energy_uJ(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
@@ -1903,12 +2023,16 @@ static int i915_energy_uJ(struct seq_file *m, void *data)
         if (INTEL_INFO(dev)->gen < 6)
                 return -ENODEV;
 
+        intel_runtime_pm_get(dev_priv);
+
         rdmsrl(MSR_RAPL_POWER_UNIT, power);
         power = (power & 0x1f00) >> 8;
         units = 1000000 / (1 << power); /* convert to uJ */
         power = I915_READ(MCH_SECP_NRG_STTS);
         power *= units;
 
+        intel_runtime_pm_put(dev_priv);
+
         seq_printf(m, "%llu", (long long unsigned)power);
 
         return 0;
@@ -1925,15 +2049,9 @@ static int i915_pc8_status(struct seq_file *m, void *unused)
                 return 0;
         }
 
-        mutex_lock(&dev_priv->pc8.lock);
-        seq_printf(m, "Requirements met: %s\n",
-                   yesno(dev_priv->pc8.requirements_met));
-        seq_printf(m, "GPU idle: %s\n", yesno(dev_priv->pc8.gpu_idle));
-        seq_printf(m, "Disable count: %d\n", dev_priv->pc8.disable_count);
+        seq_printf(m, "GPU idle: %s\n", yesno(!dev_priv->mm.busy));
         seq_printf(m, "IRQs disabled: %s\n",
-                   yesno(dev_priv->pc8.irqs_disabled));
-        seq_printf(m, "Enabled: %s\n", yesno(dev_priv->pc8.enabled));
-        mutex_unlock(&dev_priv->pc8.lock);
+                   yesno(dev_priv->pm.irqs_disabled));
 
         return 0;
 }
@@ -1961,6 +2079,28 @@ static const char *power_domain_str(enum intel_display_power_domain domain)
                 return "TRANSCODER_C";
         case POWER_DOMAIN_TRANSCODER_EDP:
                 return "TRANSCODER_EDP";
+        case POWER_DOMAIN_PORT_DDI_A_2_LANES:
+                return "PORT_DDI_A_2_LANES";
+        case POWER_DOMAIN_PORT_DDI_A_4_LANES:
+                return "PORT_DDI_A_4_LANES";
+        case POWER_DOMAIN_PORT_DDI_B_2_LANES:
+                return "PORT_DDI_B_2_LANES";
+        case POWER_DOMAIN_PORT_DDI_B_4_LANES:
+                return "PORT_DDI_B_4_LANES";
+        case POWER_DOMAIN_PORT_DDI_C_2_LANES:
+                return "PORT_DDI_C_2_LANES";
+        case POWER_DOMAIN_PORT_DDI_C_4_LANES:
+                return "PORT_DDI_C_4_LANES";
+        case POWER_DOMAIN_PORT_DDI_D_2_LANES:
+                return "PORT_DDI_D_2_LANES";
+        case POWER_DOMAIN_PORT_DDI_D_4_LANES:
+                return "PORT_DDI_D_4_LANES";
+        case POWER_DOMAIN_PORT_DSI:
+                return "PORT_DSI";
+        case POWER_DOMAIN_PORT_CRT:
+                return "PORT_CRT";
+        case POWER_DOMAIN_PORT_OTHER:
+                return "PORT_OTHER";
         case POWER_DOMAIN_VGA:
                 return "VGA";
         case POWER_DOMAIN_AUDIO:
@@ -2008,6 +2148,215 @@ static int i915_power_domain_info(struct seq_file *m, void *unused)
         return 0;
 }
 
+static void intel_seq_print_mode(struct seq_file *m, int tabs,
+                                 struct drm_display_mode *mode)
+{
+        int i;
+
+        for (i = 0; i < tabs; i++)
+                seq_putc(m, '\t');
+
+        seq_printf(m, "id %d:\"%s\" freq %d clock %d hdisp %d hss %d hse %d htot %d vdisp %d vss %d vse %d vtot %d type 0x%x flags 0x%x\n",
+                   mode->base.id, mode->name,
+                   mode->vrefresh, mode->clock,
+                   mode->hdisplay, mode->hsync_start,
+                   mode->hsync_end, mode->htotal,
+                   mode->vdisplay, mode->vsync_start,
+                   mode->vsync_end, mode->vtotal,
+                   mode->type, mode->flags);
+}
+
+static void intel_encoder_info(struct seq_file *m,
+                               struct intel_crtc *intel_crtc,
+                               struct intel_encoder *intel_encoder)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct drm_crtc *crtc = &intel_crtc->base;
+        struct intel_connector *intel_connector;
+        struct drm_encoder *encoder;
+
+        encoder = &intel_encoder->base;
+        seq_printf(m, "\tencoder %d: type: %s, connectors:\n",
+                   encoder->base.id, drm_get_encoder_name(encoder));
+        for_each_connector_on_encoder(dev, encoder, intel_connector) {
+                struct drm_connector *connector = &intel_connector->base;
+                seq_printf(m, "\t\tconnector %d: type: %s, status: %s",
+                           connector->base.id,
+                           drm_get_connector_name(connector),
+                           drm_get_connector_status_name(connector->status));
+                if (connector->status == connector_status_connected) {
+                        struct drm_display_mode *mode = &crtc->mode;
+                        seq_printf(m, ", mode:\n");
+                        intel_seq_print_mode(m, 2, mode);
+                } else {
+                        seq_putc(m, '\n');
+                }
+        }
+}
+
+static void intel_crtc_info(struct seq_file *m, struct intel_crtc *intel_crtc)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct drm_crtc *crtc = &intel_crtc->base;
+        struct intel_encoder *intel_encoder;
+
+        seq_printf(m, "\tfb: %d, pos: %dx%d, size: %dx%d\n",
+                   crtc->primary->fb->base.id, crtc->x, crtc->y,
+                   crtc->primary->fb->width, crtc->primary->fb->height);
+        for_each_encoder_on_crtc(dev, crtc, intel_encoder)
+                intel_encoder_info(m, intel_crtc, intel_encoder);
+}
+
+static void intel_panel_info(struct seq_file *m, struct intel_panel *panel)
+{
+        struct drm_display_mode *mode = panel->fixed_mode;
+
+        seq_printf(m, "\tfixed mode:\n");
+        intel_seq_print_mode(m, 2, mode);
+}
+
+static void intel_dp_info(struct seq_file *m,
+                          struct intel_connector *intel_connector)
+{
+        struct intel_encoder *intel_encoder = intel_connector->encoder;
+        struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base);
+
+        seq_printf(m, "\tDPCD rev: %x\n", intel_dp->dpcd[DP_DPCD_REV]);
+        seq_printf(m, "\taudio support: %s\n", intel_dp->has_audio ? "yes" :
+                   "no");
+        if (intel_encoder->type == INTEL_OUTPUT_EDP)
+                intel_panel_info(m, &intel_connector->panel);
+}
+
+static void intel_hdmi_info(struct seq_file *m,
+                            struct intel_connector *intel_connector)
+{
+        struct intel_encoder *intel_encoder = intel_connector->encoder;
+        struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base);
+
+        seq_printf(m, "\taudio support: %s\n", intel_hdmi->has_audio ? "yes" :
+                   "no");
+}
+
+static void intel_lvds_info(struct seq_file *m,
+                            struct intel_connector *intel_connector)
+{
+        intel_panel_info(m, &intel_connector->panel);
+}
+
+static void intel_connector_info(struct seq_file *m,
+                                 struct drm_connector *connector)
+{
+        struct intel_connector *intel_connector = to_intel_connector(connector);
+        struct intel_encoder *intel_encoder = intel_connector->encoder;
+        struct drm_display_mode *mode;
+
+        seq_printf(m, "connector %d: type %s, status: %s\n",
+                   connector->base.id, drm_get_connector_name(connector),
+                   drm_get_connector_status_name(connector->status));
+        if (connector->status == connector_status_connected) {
+                seq_printf(m, "\tname: %s\n", connector->display_info.name);
+                seq_printf(m, "\tphysical dimensions: %dx%dmm\n",
+                           connector->display_info.width_mm,
+                           connector->display_info.height_mm);
+                seq_printf(m, "\tsubpixel order: %s\n",
+                           drm_get_subpixel_order_name(connector->display_info.subpixel_order));
+                seq_printf(m, "\tCEA rev: %d\n",
+                           connector->display_info.cea_rev);
+        }
+        if (intel_encoder->type == INTEL_OUTPUT_DISPLAYPORT ||
+            intel_encoder->type == INTEL_OUTPUT_EDP)
+                intel_dp_info(m, intel_connector);
+        else if (intel_encoder->type == INTEL_OUTPUT_HDMI)
+                intel_hdmi_info(m, intel_connector);
+        else if (intel_encoder->type == INTEL_OUTPUT_LVDS)
+                intel_lvds_info(m, intel_connector);
+
+        seq_printf(m, "\tmodes:\n");
+        list_for_each_entry(mode, &connector->modes, head)
+                intel_seq_print_mode(m, 2, mode);
+}
+
+static bool cursor_active(struct drm_device *dev, int pipe)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 state;
+
+        if (IS_845G(dev) || IS_I865G(dev))
+                state = I915_READ(_CURACNTR) & CURSOR_ENABLE;
+        else if (INTEL_INFO(dev)->gen <= 6 || IS_VALLEYVIEW(dev))
+                state = I915_READ(CURCNTR(pipe)) & CURSOR_MODE;
+        else
+                state = I915_READ(CURCNTR_IVB(pipe)) & CURSOR_MODE;
+
+        return state;
+}
+
+static bool cursor_position(struct drm_device *dev, int pipe, int *x, int *y)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 pos;
+
+        if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_BROADWELL(dev))
+                pos = I915_READ(CURPOS_IVB(pipe));
+        else
+                pos = I915_READ(CURPOS(pipe));
+
+        *x = (pos >> CURSOR_X_SHIFT) & CURSOR_POS_MASK;
+        if (pos & (CURSOR_POS_SIGN << CURSOR_X_SHIFT))
+                *x = -*x;
+
+        *y = (pos >> CURSOR_Y_SHIFT) & CURSOR_POS_MASK;
+        if (pos & (CURSOR_POS_SIGN << CURSOR_Y_SHIFT))
+                *y = -*y;
+
+        return cursor_active(dev, pipe);
+}
+
+static int i915_display_info(struct seq_file *m, void *unused)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *crtc;
+        struct drm_connector *connector;
+
+        intel_runtime_pm_get(dev_priv);
+        drm_modeset_lock_all(dev);
+        seq_printf(m, "CRTC info\n");
+        seq_printf(m, "---------\n");
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
+                bool active;
+                int x, y;
+
+                seq_printf(m, "CRTC %d: pipe: %c, active: %s\n",
+                           crtc->base.base.id, pipe_name(crtc->pipe),
+                           yesno(crtc->active));
+                if (crtc->active) {
+                        intel_crtc_info(m, crtc);
+
+                        active = cursor_position(dev, crtc->pipe, &x, &y);
+                        seq_printf(m, "\tcursor visible? %s, position (%d, %d), addr 0x%08x, active? %s\n",
+                                   yesno(crtc->cursor_visible),
+                                   x, y, crtc->cursor_addr,
+                                   yesno(active));
+                }
+        }
+
+        seq_printf(m, "\n");
+        seq_printf(m, "Connector info\n");
+        seq_printf(m, "--------------\n");
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                intel_connector_info(m, connector);
+        }
+        drm_modeset_unlock_all(dev);
+        intel_runtime_pm_put(dev_priv);
+
+        return 0;
+}
+
 struct pipe_crc_info {
         const char *name;
         struct drm_device *dev;
@@ -2332,8 +2681,6 @@ static int vlv_pipe_crc_ctl_reg(struct drm_device *dev,
         if (need_stable_symbols) {
                 uint32_t tmp = I915_READ(PORT_DFT2_G4X);
 
-                WARN_ON(!IS_G4X(dev));
-
                 tmp |= DC_BALANCE_RESET_VLV;
                 if (pipe == PIPE_A)
                         tmp |= PIPE_A_SCRAMBLE_RESET;
@@ -2756,11 +3103,179 @@ static const struct file_operations i915_display_crc_ctl_fops = {
         .write = display_crc_ctl_write
 };
 
+static void wm_latency_show(struct seq_file *m, const uint16_t wm[5])
+{
+        struct drm_device *dev = m->private;
+        int num_levels = IS_HASWELL(dev) || IS_BROADWELL(dev) ? 5 : 4;
+        int level;
+
+        drm_modeset_lock_all(dev);
+
+        for (level = 0; level < num_levels; level++) {
+                unsigned int latency = wm[level];
+
+                /* WM1+ latency values in 0.5us units */
+                if (level > 0)
+                        latency *= 5;
+
+                seq_printf(m, "WM%d %u (%u.%u usec)\n",
+                           level, wm[level],
+                           latency / 10, latency % 10);
+        }
+
+        drm_modeset_unlock_all(dev);
+}
+
+static int pri_wm_latency_show(struct seq_file *m, void *data)
+{
+        struct drm_device *dev = m->private;
+
+        wm_latency_show(m, to_i915(dev)->wm.pri_latency);
+
+        return 0;
+}
+
+static int spr_wm_latency_show(struct seq_file *m, void *data)
+{
+        struct drm_device *dev = m->private;
+
+        wm_latency_show(m, to_i915(dev)->wm.spr_latency);
+
+        return 0;
+}
+
+static int cur_wm_latency_show(struct seq_file *m, void *data)
+{
+        struct drm_device *dev = m->private;
+
+        wm_latency_show(m, to_i915(dev)->wm.cur_latency);
+
+        return 0;
+}
+
+static int pri_wm_latency_open(struct inode *inode, struct file *file)
+{
+        struct drm_device *dev = inode->i_private;
+
+        if (!HAS_PCH_SPLIT(dev))
+                return -ENODEV;
+
+        return single_open(file, pri_wm_latency_show, dev);
+}
+
+static int spr_wm_latency_open(struct inode *inode, struct file *file)
+{
+        struct drm_device *dev = inode->i_private;
+
+        if (!HAS_PCH_SPLIT(dev))
+                return -ENODEV;
+
+        return single_open(file, spr_wm_latency_show, dev);
+}
+
+static int cur_wm_latency_open(struct inode *inode, struct file *file)
+{
+        struct drm_device *dev = inode->i_private;
+
+        if (!HAS_PCH_SPLIT(dev))
+                return -ENODEV;
+
+        return single_open(file, cur_wm_latency_show, dev);
+}
+
+static ssize_t wm_latency_write(struct file *file, const char __user *ubuf,
+                                size_t len, loff_t *offp, uint16_t wm[5])
+{
+        struct seq_file *m = file->private_data;
+        struct drm_device *dev = m->private;
+        uint16_t new[5] = { 0 };
+        int num_levels = IS_HASWELL(dev) || IS_BROADWELL(dev) ? 5 : 4;
+        int level;
+        int ret;
+        char tmp[32];
+
+        if (len >= sizeof(tmp))
+                return -EINVAL;
+
+        if (copy_from_user(tmp, ubuf, len))
+                return -EFAULT;
+
+        tmp[len] = '\0';
+
+        ret = sscanf(tmp, "%hu %hu %hu %hu %hu", &new[0], &new[1], &new[2], &new[3], &new[4]);
+        if (ret != num_levels)
+                return -EINVAL;
+
+        drm_modeset_lock_all(dev);
+
+        for (level = 0; level < num_levels; level++)
+                wm[level] = new[level];
+
+        drm_modeset_unlock_all(dev);
+
+        return len;
+}
+
+
+static ssize_t pri_wm_latency_write(struct file *file, const char __user *ubuf,
+                                    size_t len, loff_t *offp)
+{
+        struct seq_file *m = file->private_data;
+        struct drm_device *dev = m->private;
+
+        return wm_latency_write(file, ubuf, len, offp, to_i915(dev)->wm.pri_latency);
+}
+
+static ssize_t spr_wm_latency_write(struct file *file, const char __user *ubuf,
+                                    size_t len, loff_t *offp)
+{
+        struct seq_file *m = file->private_data;
+        struct drm_device *dev = m->private;
+
+        return wm_latency_write(file, ubuf, len, offp, to_i915(dev)->wm.spr_latency);
+}
+
+static ssize_t cur_wm_latency_write(struct file *file, const char __user *ubuf,
+                                    size_t len, loff_t *offp)
+{
+        struct seq_file *m = file->private_data;
+        struct drm_device *dev = m->private;
+
+        return wm_latency_write(file, ubuf, len, offp, to_i915(dev)->wm.cur_latency);
+}
+
+static const struct file_operations i915_pri_wm_latency_fops = {
+        .owner = THIS_MODULE,
+        .open = pri_wm_latency_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+        .write = pri_wm_latency_write
+};
+
+static const struct file_operations i915_spr_wm_latency_fops = {
+        .owner = THIS_MODULE,
+        .open = spr_wm_latency_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+        .write = spr_wm_latency_write
+};
+
+static const struct file_operations i915_cur_wm_latency_fops = {
+        .owner = THIS_MODULE,
+        .open = cur_wm_latency_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
+        .write = cur_wm_latency_write
+};
+
 static int
 i915_wedged_get(void *data, u64 *val)
 {
         struct drm_device *dev = data;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         *val = atomic_read(&dev_priv->gpu_error.reset_counter);
 
@@ -2772,9 +3287,8 @@ i915_wedged_set(void *data, u64 val)
 {
         struct drm_device *dev = data;
 
-        DRM_INFO("Manually setting wedged to %llu\n", val);
-        i915_handle_error(dev, val);
-
+        i915_handle_error(dev, val,
+                          "Manually setting wedged to %llu", val);
         return 0;
 }
 
@@ -2786,7 +3300,7 @@ static int
 i915_ring_stop_get(void *data, u64 *val)
 {
         struct drm_device *dev = data;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         *val = dev_priv->gpu_error.stop_rings;
 
@@ -2929,7 +3443,7 @@ i915_drop_caches_set(void *data, u64 val)
                 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
                         list_for_each_entry_safe(vma, x, &vm->inactive_list,
                                                  mm_list) {
-                                if (vma->obj->pin_count)
+                                if (vma->pin_count)
                                         continue;
 
                                 ret = i915_vma_unbind(vma);
@@ -2963,7 +3477,7 @@ static int
 i915_max_freq_get(void *data, u64 *val)
 {
         struct drm_device *dev = data;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
         if (!(IS_GEN6(dev) || IS_GEN7(dev)))
@@ -2976,9 +3490,9 @@ i915_max_freq_get(void *data, u64 *val)
                 return ret;
 
         if (IS_VALLEYVIEW(dev))
-                *val = vlv_gpu_freq(dev_priv, dev_priv->rps.max_delay);
+                *val = vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq_softlimit);
         else
-                *val = dev_priv->rps.max_delay * GT_FREQUENCY_MULTIPLIER;
+                *val = dev_priv->rps.max_freq_softlimit * GT_FREQUENCY_MULTIPLIER;
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         return 0;
@@ -2989,6 +3503,7 @@ i915_max_freq_set(void *data, u64 val)
 {
         struct drm_device *dev = data;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 rp_state_cap, hw_max, hw_min;
         int ret;
 
         if (!(IS_GEN6(dev) || IS_GEN7(dev)))
@@ -3007,14 +3522,29 @@ i915_max_freq_set(void *data, u64 val)
          */
         if (IS_VALLEYVIEW(dev)) {
                 val = vlv_freq_opcode(dev_priv, val);
-                dev_priv->rps.max_delay = val;
-                valleyview_set_rps(dev, val);
+
+                hw_max = valleyview_rps_max_freq(dev_priv);
+                hw_min = valleyview_rps_min_freq(dev_priv);
         } else {
                 do_div(val, GT_FREQUENCY_MULTIPLIER);
-                dev_priv->rps.max_delay = val;
-                gen6_set_rps(dev, val);
+
+                rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
+                hw_max = dev_priv->rps.max_freq;
+                hw_min = (rp_state_cap >> 16) & 0xff;
+        }
+
+        if (val < hw_min || val > hw_max || val < dev_priv->rps.min_freq_softlimit) {
+                mutex_unlock(&dev_priv->rps.hw_lock);
+                return -EINVAL;
         }
 
+        dev_priv->rps.max_freq_softlimit = val;
+
+        if (IS_VALLEYVIEW(dev))
+                valleyview_set_rps(dev, val);
+        else
+                gen6_set_rps(dev, val);
+
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         return 0;
@@ -3028,7 +3558,7 @@ static int
 i915_min_freq_get(void *data, u64 *val)
 {
         struct drm_device *dev = data;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
         if (!(IS_GEN6(dev) || IS_GEN7(dev)))
@@ -3041,9 +3571,9 @@ i915_min_freq_get(void *data, u64 *val)
                 return ret;
 
         if (IS_VALLEYVIEW(dev))
-                *val = vlv_gpu_freq(dev_priv, dev_priv->rps.min_delay);
+                *val = vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq_softlimit);
         else
-                *val = dev_priv->rps.min_delay * GT_FREQUENCY_MULTIPLIER;
+                *val = dev_priv->rps.min_freq_softlimit * GT_FREQUENCY_MULTIPLIER;
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         return 0;
@@ -3054,6 +3584,7 @@ i915_min_freq_set(void *data, u64 val)
 {
         struct drm_device *dev = data;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 rp_state_cap, hw_max, hw_min;
         int ret;
 
         if (!(IS_GEN6(dev) || IS_GEN7(dev)))
@@ -3072,13 +3603,29 @@ i915_min_freq_set(void *data, u64 val)
          */
         if (IS_VALLEYVIEW(dev)) {
                 val = vlv_freq_opcode(dev_priv, val);
-                dev_priv->rps.min_delay = val;
-                valleyview_set_rps(dev, val);
+
+                hw_max = valleyview_rps_max_freq(dev_priv);
+                hw_min = valleyview_rps_min_freq(dev_priv);
         } else {
                 do_div(val, GT_FREQUENCY_MULTIPLIER);
-                dev_priv->rps.min_delay = val;
-                gen6_set_rps(dev, val);
+
+                rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
+                hw_max = dev_priv->rps.max_freq;
+                hw_min = (rp_state_cap >> 16) & 0xff;
+        }
+
+        if (val < hw_min || val > hw_max || val > dev_priv->rps.max_freq_softlimit) {
+                mutex_unlock(&dev_priv->rps.hw_lock);
+                return -EINVAL;
         }
+
+        dev_priv->rps.min_freq_softlimit = val;
+
+        if (IS_VALLEYVIEW(dev))
+                valleyview_set_rps(dev, val);
+        else
+                gen6_set_rps(dev, val);
+
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         return 0;
@@ -3092,7 +3639,7 @@ static int
 i915_cache_sharing_get(void *data, u64 *val)
 {
         struct drm_device *dev = data;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 snpcr;
         int ret;
 
@@ -3152,7 +3699,6 @@ static int i915_forcewake_open(struct inode *inode, struct file *file)
         if (INTEL_INFO(dev)->gen < 6)
                 return 0;
 
-        intel_runtime_pm_get(dev_priv);
         gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
 
         return 0;
@@ -3167,7 +3713,6 @@ static int i915_forcewake_release(struct inode *inode, struct file *file)
                 return 0;
 
         gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
-        intel_runtime_pm_put(dev_priv);
 
         return 0;
 }
@@ -3248,9 +3793,11 @@ static const struct drm_info_list i915_debugfs_list[] = {
         {"i915_dpio", i915_dpio_info, 0},
         {"i915_llc", i915_llc, 0},
         {"i915_edp_psr_status", i915_edp_psr_status, 0},
+        {"i915_sink_crc_eDP1", i915_sink_crc, 0},
         {"i915_energy_uJ", i915_energy_uJ, 0},
         {"i915_pc8_status", i915_pc8_status, 0},
         {"i915_power_domain_info", i915_power_domain_info, 0},
+        {"i915_display_info", i915_display_info, 0},
 };
 #define I915_DEBUGFS_ENTRIES ARRAY_SIZE(i915_debugfs_list)
 
@@ -3269,6 +3816,9 @@ static const struct i915_debugfs_files {
         {"i915_error_state", &i915_error_state_fops},
         {"i915_next_seqno", &i915_next_seqno_fops},
         {"i915_display_crc_ctl", &i915_display_crc_ctl_fops},
+        {"i915_pri_wm_latency", &i915_pri_wm_latency_fops},
+        {"i915_spr_wm_latency", &i915_spr_wm_latency_fops},
+        {"i915_cur_wm_latency", &i915_cur_wm_latency_fops},
 };
 
 void intel_display_crc_init(struct drm_device *dev)
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index 15a74f979b4b..96177eec0a0e 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -82,7 +82,7 @@ intel_read_legacy_status_page(struct drm_i915_private *dev_priv, int reg)
 
 void i915_update_dri1_breadcrumb(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_master_private *master_priv;
 
         /*
@@ -103,7 +103,7 @@ void i915_update_dri1_breadcrumb(struct drm_device *dev)
 
 static void i915_write_hws_pga(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 addr;
 
         addr = dev_priv->status_page_dmah->busaddr;
@@ -118,7 +118,7 @@ static void i915_write_hws_pga(struct drm_device *dev)
  */
 static void i915_free_hws(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring = LP_RING(dev_priv);
 
         if (dev_priv->status_page_dmah) {
@@ -137,7 +137,7 @@ static void i915_free_hws(struct drm_device *dev)
 
 void i915_kernel_lost_context(struct drm_device * dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_master_private *master_priv;
         struct intel_ring_buffer *ring = LP_RING(dev_priv);
 
@@ -164,7 +164,7 @@ void i915_kernel_lost_context(struct drm_device * dev)
 
 static int i915_dma_cleanup(struct drm_device * dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int i;
 
         /* Make sure interrupts are disabled here because the uninstall ioctl
@@ -188,7 +188,7 @@ static int i915_dma_cleanup(struct drm_device * dev)
 
 static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
         int ret;
 
@@ -233,7 +233,7 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
 
 static int i915_dma_resume(struct drm_device * dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring = LP_RING(dev_priv);
 
         DRM_DEBUG_DRIVER("%s\n", __func__);
@@ -357,7 +357,7 @@ static int validate_cmd(int cmd)
 
 static int i915_emit_cmds(struct drm_device * dev, int *buffer, int dwords)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int i, ret;
 
         if ((dwords+1) * sizeof(int) >= LP_RING(dev_priv)->size - 8)
@@ -431,7 +431,7 @@ i915_emit_box(struct drm_device *dev,
 
 static void i915_emit_breadcrumb(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 
         dev_priv->dri1.counter++;
@@ -547,7 +547,7 @@ static int i915_dispatch_batchbuffer(struct drm_device * dev,
 
 static int i915_dispatch_flip(struct drm_device * dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_master_private *master_priv =
                 dev->primary->master->driver_priv;
         int ret;
@@ -625,10 +625,9 @@ static int i915_flush_ioctl(struct drm_device *dev, void *data,
 static int i915_batchbuffer(struct drm_device *dev, void *data,
                             struct drm_file *file_priv)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-        struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
-        drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
-            master_priv->sarea_priv;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_master_private *master_priv;
+        drm_i915_sarea_t *sarea_priv;
         drm_i915_batchbuffer_t *batch = data;
         int ret;
         struct drm_clip_rect *cliprects = NULL;
@@ -636,6 +635,9 @@ static int i915_batchbuffer(struct drm_device *dev, void *data,
         if (drm_core_check_feature(dev, DRIVER_MODESET))
                 return -ENODEV;
 
+        master_priv = dev->primary->master->driver_priv;
+        sarea_priv = (drm_i915_sarea_t *) master_priv->sarea_priv;
+
         if (!dev_priv->dri1.allow_batchbuffer) {
                 DRM_ERROR("Batchbuffer ioctl disabled\n");
                 return -EINVAL;
@@ -681,10 +683,9 @@ fail_free:
 static int i915_cmdbuffer(struct drm_device *dev, void *data,
                           struct drm_file *file_priv)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-        struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
-        drm_i915_sarea_t *sarea_priv = (drm_i915_sarea_t *)
-            master_priv->sarea_priv;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_master_private *master_priv;
+        drm_i915_sarea_t *sarea_priv;
         drm_i915_cmdbuffer_t *cmdbuf = data;
         struct drm_clip_rect *cliprects = NULL;
         void *batch_data;
@@ -696,6 +697,9 @@ static int i915_cmdbuffer(struct drm_device *dev, void *data,
         if (drm_core_check_feature(dev, DRIVER_MODESET))
                 return -ENODEV;
 
+        master_priv = dev->primary->master->driver_priv;
+        sarea_priv = (drm_i915_sarea_t *) master_priv->sarea_priv;
+
         RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 
         if (cmdbuf->num_cliprects < 0)
@@ -749,7 +753,7 @@ fail_batch_free:
 
 static int i915_emit_irq(struct drm_device * dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
 
         i915_kernel_lost_context(dev);
@@ -775,7 +779,7 @@ static int i915_emit_irq(struct drm_device * dev)
 
 static int i915_wait_irq(struct drm_device * dev, int irq_nr)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
         int ret = 0;
         struct intel_ring_buffer *ring = LP_RING(dev_priv);
@@ -812,7 +816,7 @@ static int i915_wait_irq(struct drm_device * dev, int irq_nr)
 static int i915_irq_emit(struct drm_device *dev, void *data,
                          struct drm_file *file_priv)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         drm_i915_irq_emit_t *emit = data;
         int result;
 
@@ -843,7 +847,7 @@ static int i915_irq_emit(struct drm_device *dev, void *data,
 static int i915_irq_wait(struct drm_device *dev, void *data,
                          struct drm_file *file_priv)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         drm_i915_irq_wait_t *irqwait = data;
 
         if (drm_core_check_feature(dev, DRIVER_MODESET))
@@ -860,7 +864,7 @@ static int i915_irq_wait(struct drm_device *dev, void *data,
 static int i915_vblank_pipe_get(struct drm_device *dev, void *data,
                          struct drm_file *file_priv)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         drm_i915_vblank_pipe_t *pipe = data;
 
         if (drm_core_check_feature(dev, DRIVER_MODESET))
@@ -921,7 +925,7 @@ static int i915_flip_bufs(struct drm_device *dev, void *data,
 static int i915_getparam(struct drm_device *dev, void *data,
                          struct drm_file *file_priv)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         drm_i915_getparam_t *param = data;
         int value;
 
@@ -990,7 +994,7 @@ static int i915_getparam(struct drm_device *dev, void *data,
                 value = HAS_WT(dev);
                 break;
         case I915_PARAM_HAS_ALIASING_PPGTT:
-                value = dev_priv->mm.aliasing_ppgtt ? 1 : 0;
+                value = dev_priv->mm.aliasing_ppgtt || USES_FULL_PPGTT(dev);
                 break;
         case I915_PARAM_HAS_WAIT_TIMEOUT:
                 value = 1;
@@ -1029,7 +1033,7 @@ static int i915_getparam(struct drm_device *dev, void *data,
 static int i915_setparam(struct drm_device *dev, void *data,
                          struct drm_file *file_priv)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         drm_i915_setparam_t *param = data;
 
         if (!dev_priv) {
@@ -1064,7 +1068,7 @@ static int i915_setparam(struct drm_device *dev, void *data,
 static int i915_set_status_page(struct drm_device *dev, void *data,
                                 struct drm_file *file_priv)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         drm_i915_hws_addr_t *hws = data;
         struct intel_ring_buffer *ring;
 
@@ -1132,7 +1136,7 @@ static int i915_get_bridge_dev(struct drm_device *dev)
 static int
 intel_alloc_mchbar_resource(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
         u32 temp_lo, temp_hi = 0;
         u64 mchbar_addr;
@@ -1178,11 +1182,14 @@ intel_alloc_mchbar_resource(struct drm_device *dev)
 static void
 intel_setup_mchbar(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
         u32 temp;
         bool enabled;
 
+        if (IS_VALLEYVIEW(dev))
+                return;
+
         dev_priv->mchbar_need_disable = false;
 
         if (IS_I915G(dev) || IS_I915GM(dev)) {
@@ -1215,7 +1222,7 @@ intel_setup_mchbar(struct drm_device *dev)
 static void
 intel_teardown_mchbar(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
         u32 temp;
 
@@ -1317,12 +1324,12 @@ static int i915_load_modeset_init(struct drm_device *dev)
         if (ret)
                 goto cleanup_vga_switcheroo;
 
+        intel_power_domains_init_hw(dev_priv);
+
         ret = drm_irq_install(dev);
         if (ret)
                 goto cleanup_gem_stolen;
 
-        intel_power_domains_init_hw(dev);
-
         /* Important: The output setup functions called by modeset_init need
          * working irqs for e.g. gmbus and dp aux transfers. */
         intel_modeset_init(dev);
@@ -1339,7 +1346,7 @@ static int i915_load_modeset_init(struct drm_device *dev)
         /* FIXME: do pre/post-mode set stuff in core KMS code */
         dev->vblank_disable_allowed = true;
         if (INTEL_INFO(dev)->num_pipes == 0) {
-                intel_display_power_put(dev, POWER_DOMAIN_VGA);
+                intel_display_power_put(dev_priv, POWER_DOMAIN_VGA);
                 return 0;
         }
 
@@ -1374,10 +1381,10 @@ cleanup_gem:
         i915_gem_cleanup_ringbuffer(dev);
         i915_gem_context_fini(dev);
         mutex_unlock(&dev->struct_mutex);
-        i915_gem_cleanup_aliasing_ppgtt(dev);
+        WARN_ON(dev_priv->mm.aliasing_ppgtt);
         drm_mm_takedown(&dev_priv->gtt.base.mm);
 cleanup_power:
-        intel_display_power_put(dev, POWER_DOMAIN_VGA);
+        intel_display_power_put(dev_priv, POWER_DOMAIN_VGA);
         drm_irq_uninstall(dev);
 cleanup_gem_stolen:
         i915_gem_cleanup_stolen(dev);
@@ -1442,7 +1449,7 @@ static void i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
 
 static void i915_dump_device_info(struct drm_i915_private *dev_priv)
 {
-        const struct intel_device_info *info = dev_priv->info;
+        const struct intel_device_info *info = &dev_priv->info;
 
 #define PRINT_S(name) "%s"
 #define SEP_EMPTY
@@ -1459,6 +1466,62 @@ static void i915_dump_device_info(struct drm_i915_private *dev_priv)
 #undef SEP_COMMA
 }
 
+/*
+ * Determine various intel_device_info fields at runtime.
+ *
+ * Use it when either:
+ *   - it's judged too laborious to fill n static structures with the limit
+ *     when a simple if statement does the job,
+ *   - run-time checks (eg read fuse/strap registers) are needed.
+ *
+ * This function needs to be called:
+ *   - after the MMIO has been setup as we are reading registers,
+ *   - after the PCH has been detected,
+ *   - before the first usage of the fields it can tweak.
+ */
+static void intel_device_info_runtime_init(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_device_info *info;
+        enum pipe pipe;
+
+        info = (struct intel_device_info *)&dev_priv->info;
+
+        if (IS_VALLEYVIEW(dev))
+                for_each_pipe(pipe)
+                        info->num_sprites[pipe] = 2;
+        else
+                for_each_pipe(pipe)
+                        info->num_sprites[pipe] = 1;
+
+        if (i915.disable_display) {
+                DRM_INFO("Display disabled (module parameter)\n");
+                info->num_pipes = 0;
+        } else if (info->num_pipes > 0 &&
+                   (INTEL_INFO(dev)->gen == 7 || INTEL_INFO(dev)->gen == 8) &&
+                   !IS_VALLEYVIEW(dev)) {
+                u32 fuse_strap = I915_READ(FUSE_STRAP);
+                u32 sfuse_strap = I915_READ(SFUSE_STRAP);
+
+                /*
+                 * SFUSE_STRAP is supposed to have a bit signalling the display
+                 * is fused off. Unfortunately it seems that, at least in
+                 * certain cases, fused off display means that PCH display
+                 * reads don't land anywhere. In that case, we read 0s.
+                 *
+                 * On CPT/PPT, we can detect this case as SFUSE_STRAP_FUSE_LOCK
+                 * should be set when taking over after the firmware.
+                 */
+                if (fuse_strap & ILK_INTERNAL_DISPLAY_DISABLE ||
+                    sfuse_strap & SFUSE_STRAP_DISPLAY_DISABLED ||
+                    (dev_priv->pch_type == PCH_CPT &&
+                     !(sfuse_strap & SFUSE_STRAP_FUSE_LOCK))) {
+                        DRM_INFO("Display fused off, disabling\n");
+                        info->num_pipes = 0;
+                }
+        }
+}
+
 /**
  * i915_driver_load - setup chip and create an initial config
  * @dev: DRM device
@@ -1473,7 +1536,7 @@ static void i915_dump_device_info(struct drm_i915_private *dev_priv)
 int i915_driver_load(struct drm_device *dev, unsigned long flags)
 {
         struct drm_i915_private *dev_priv;
-        struct intel_device_info *info;
+        struct intel_device_info *info, *device_info;
         int ret = 0, mmio_bar, mmio_size;
         uint32_t aperture_size;
 
@@ -1496,7 +1559,10 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
 
         dev->dev_private = (void *)dev_priv;
         dev_priv->dev = dev;
-        dev_priv->info = info;
+
+        /* copy initial configuration to dev_priv->info */
+        device_info = (struct intel_device_info *)&dev_priv->info;
+        *device_info = *info;
 
         spin_lock_init(&dev_priv->irq_lock);
         spin_lock_init(&dev_priv->gpu_error.lock);
@@ -1545,8 +1611,6 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
                 goto put_bridge;
         }
 
-        intel_uncore_early_sanitize(dev);
-
         /* This must be called before any calls to HAS_PCH_* */
         intel_detect_pch(dev);
 
@@ -1635,9 +1699,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         if (!IS_I945G(dev) && !IS_I945GM(dev))
                 pci_enable_msi(dev->pdev);
 
-        dev_priv->num_plane = 1;
-        if (IS_VALLEYVIEW(dev))
-                dev_priv->num_plane = 2;
+        intel_device_info_runtime_init(dev);
 
         if (INTEL_INFO(dev)->num_pipes) {
                 ret = drm_vblank_init(dev, INTEL_INFO(dev)->num_pipes);
@@ -1645,7 +1707,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
                         goto out_gem_unload;
         }
 
-        intel_power_domains_init(dev);
+        intel_power_domains_init(dev_priv);
 
         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
                 ret = i915_load_modeset_init(dev);
@@ -1674,7 +1736,7 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         return 0;
 
 out_power_well:
-        intel_power_domains_remove(dev);
+        intel_power_domains_remove(dev_priv);
         drm_vblank_cleanup(dev);
 out_gem_unload:
         if (dev_priv->mm.inactive_shrinker.scan_objects)
@@ -1724,8 +1786,8 @@ int i915_driver_unload(struct drm_device *dev)
         /* The i915.ko module is still not prepared to be loaded when
          * the power well is not enabled, so just enable it in case
          * we're going to unload/reload. */
-        intel_display_set_init_power(dev, true);
-        intel_power_domains_remove(dev);
+        intel_display_set_init_power(dev_priv, true);
+        intel_power_domains_remove(dev_priv);
 
         i915_teardown_sysfs(dev);
 
@@ -1761,8 +1823,6 @@ int i915_driver_unload(struct drm_device *dev)
         cancel_work_sync(&dev_priv->gpu_error.work);
         i915_destroy_error_state(dev);
 
-        cancel_delayed_work_sync(&dev_priv->pc8.enable_work);
-
         if (dev->pdev->msi_enabled)
                 pci_disable_msi(dev->pdev);
 
@@ -1776,8 +1836,8 @@ int i915_driver_unload(struct drm_device *dev)
                 i915_gem_free_all_phys_object(dev);
                 i915_gem_cleanup_ringbuffer(dev);
                 i915_gem_context_fini(dev);
+                WARN_ON(dev_priv->mm.aliasing_ppgtt);
                 mutex_unlock(&dev->struct_mutex);
-                i915_gem_cleanup_aliasing_ppgtt(dev);
                 i915_gem_cleanup_stolen(dev);
 
                 if (!I915_NEED_GFX_HWS(dev))
@@ -1835,7 +1895,7 @@ int i915_driver_open(struct drm_device *dev, struct drm_file *file)
  */
 void i915_driver_lastclose(struct drm_device * dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         /* On gen6+ we refuse to init without kms enabled, but then the drm core
          * goes right around and calls lastclose. Check for this and don't clean
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index ec7bb0fc71bc..82f4d1f47d3b 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -38,134 +38,30 @@
 #include <linux/module.h>
 #include <drm/drm_crtc_helper.h>
 
-static int i915_modeset __read_mostly = -1;
-module_param_named(modeset, i915_modeset, int, 0400);
-MODULE_PARM_DESC(modeset,
-                "Use kernel modesetting [KMS] (0=DRM_I915_KMS from .config, "
-                "1=on, -1=force vga console preference [default])");
-
-unsigned int i915_fbpercrtc __always_unused = 0;
-module_param_named(fbpercrtc, i915_fbpercrtc, int, 0400);
-
-int i915_panel_ignore_lid __read_mostly = 1;
-module_param_named(panel_ignore_lid, i915_panel_ignore_lid, int, 0600);
-MODULE_PARM_DESC(panel_ignore_lid,
-                "Override lid status (0=autodetect, 1=autodetect disabled [default], "
-                "-1=force lid closed, -2=force lid open)");
-
-unsigned int i915_powersave __read_mostly = 1;
-module_param_named(powersave, i915_powersave, int, 0600);
-MODULE_PARM_DESC(powersave,
-                "Enable powersavings, fbc, downclocking, etc. (default: true)");
-
-int i915_semaphores __read_mostly = -1;
-module_param_named(semaphores, i915_semaphores, int, 0400);
-MODULE_PARM_DESC(semaphores,
-                "Use semaphores for inter-ring sync (default: -1 (use per-chip defaults))");
-
-int i915_enable_rc6 __read_mostly = -1;
-module_param_named(i915_enable_rc6, i915_enable_rc6, int, 0400);
-MODULE_PARM_DESC(i915_enable_rc6,
-                "Enable power-saving render C-state 6. "
-                "Different stages can be selected via bitmask values "
-                "(0 = disable; 1 = enable rc6; 2 = enable deep rc6; 4 = enable deepest rc6). "
-                "For example, 3 would enable rc6 and deep rc6, and 7 would enable everything. "
-                "default: -1 (use per-chip default)");
-
-int i915_enable_fbc __read_mostly = -1;
-module_param_named(i915_enable_fbc, i915_enable_fbc, int, 0600);
-MODULE_PARM_DESC(i915_enable_fbc,
-                "Enable frame buffer compression for power savings "
-                "(default: -1 (use per-chip default))");
-
-unsigned int i915_lvds_downclock __read_mostly = 0;
-module_param_named(lvds_downclock, i915_lvds_downclock, int, 0400);
-MODULE_PARM_DESC(lvds_downclock,
-                "Use panel (LVDS/eDP) downclocking for power savings "
-                "(default: false)");
-
-int i915_lvds_channel_mode __read_mostly;
-module_param_named(lvds_channel_mode, i915_lvds_channel_mode, int, 0600);
-MODULE_PARM_DESC(lvds_channel_mode,
-                 "Specify LVDS channel mode "
-                 "(0=probe BIOS [default], 1=single-channel, 2=dual-channel)");
-
-int i915_panel_use_ssc __read_mostly = -1;
-module_param_named(lvds_use_ssc, i915_panel_use_ssc, int, 0600);
-MODULE_PARM_DESC(lvds_use_ssc,
-                "Use Spread Spectrum Clock with panels [LVDS/eDP] "
-                "(default: auto from VBT)");
-
-int i915_vbt_sdvo_panel_type __read_mostly = -1;
-module_param_named(vbt_sdvo_panel_type, i915_vbt_sdvo_panel_type, int, 0600);
-MODULE_PARM_DESC(vbt_sdvo_panel_type,
-                "Override/Ignore selection of SDVO panel mode in the VBT "
-                "(-2=ignore, -1=auto [default], index in VBT BIOS table)");
-
-static bool i915_try_reset __read_mostly = true;
-module_param_named(reset, i915_try_reset, bool, 0600);
-MODULE_PARM_DESC(reset, "Attempt GPU resets (default: true)");
-
-bool i915_enable_hangcheck __read_mostly = true;
-module_param_named(enable_hangcheck, i915_enable_hangcheck, bool, 0644);
-MODULE_PARM_DESC(enable_hangcheck,
-                "Periodically check GPU activity for detecting hangs. "
-                "WARNING: Disabling this can cause system wide hangs. "
-                "(default: true)");
-
-int i915_enable_ppgtt __read_mostly = -1;
-module_param_named(i915_enable_ppgtt, i915_enable_ppgtt, int, 0400);
-MODULE_PARM_DESC(i915_enable_ppgtt,
-                "Enable PPGTT (default: true)");
-
-int i915_enable_psr __read_mostly = 0;
-module_param_named(enable_psr, i915_enable_psr, int, 0600);
-MODULE_PARM_DESC(enable_psr, "Enable PSR (default: false)");
-
-unsigned int i915_preliminary_hw_support __read_mostly = IS_ENABLED(CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT);
-module_param_named(preliminary_hw_support, i915_preliminary_hw_support, int, 0600);
-MODULE_PARM_DESC(preliminary_hw_support,
-                "Enable preliminary hardware support.");
-
-int i915_disable_power_well __read_mostly = 1;
-module_param_named(disable_power_well, i915_disable_power_well, int, 0600);
-MODULE_PARM_DESC(disable_power_well,
-                 "Disable the power well when possible (default: true)");
-
-int i915_enable_ips __read_mostly = 1;
-module_param_named(enable_ips, i915_enable_ips, int, 0600);
-MODULE_PARM_DESC(enable_ips, "Enable IPS (default: true)");
-
-bool i915_fastboot __read_mostly = 0;
-module_param_named(fastboot, i915_fastboot, bool, 0600);
-MODULE_PARM_DESC(fastboot, "Try to skip unnecessary mode sets at boot time "
-                 "(default: false)");
-
-int i915_enable_pc8 __read_mostly = 1;
-module_param_named(enable_pc8, i915_enable_pc8, int, 0600);
-MODULE_PARM_DESC(enable_pc8, "Enable support for low power package C states (PC8+) (default: true)");
-
-int i915_pc8_timeout __read_mostly = 5000;
-module_param_named(pc8_timeout, i915_pc8_timeout, int, 0600);
-MODULE_PARM_DESC(pc8_timeout, "Number of msecs of idleness required to enter PC8+ (default: 5000)");
-
-bool i915_prefault_disable __read_mostly;
-module_param_named(prefault_disable, i915_prefault_disable, bool, 0600);
-MODULE_PARM_DESC(prefault_disable,
-                "Disable page prefaulting for pread/pwrite/reloc (default:false). For developers only.");
-
 static struct drm_driver driver;
 
+#define GEN_DEFAULT_PIPEOFFSETS \
+        .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \
+                          PIPE_C_OFFSET, PIPE_EDP_OFFSET }, \
+        .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \
+                           TRANSCODER_C_OFFSET, TRANSCODER_EDP_OFFSET }, \
+        .dpll_offsets = { DPLL_A_OFFSET, DPLL_B_OFFSET }, \
+        .dpll_md_offsets = { DPLL_A_MD_OFFSET, DPLL_B_MD_OFFSET }, \
+        .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET }
+
+
 static const struct intel_device_info intel_i830_info = {
         .gen = 2, .is_mobile = 1, .cursor_needs_physical = 1, .num_pipes = 2,
         .has_overlay = 1, .overlay_needs_physical = 1,
         .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_845g_info = {
         .gen = 2, .num_pipes = 1,
         .has_overlay = 1, .overlay_needs_physical = 1,
         .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_i85x_info = {
@@ -174,18 +70,21 @@ static const struct intel_device_info intel_i85x_info = {
         .has_overlay = 1, .overlay_needs_physical = 1,
         .has_fbc = 1,
         .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_i865g_info = {
         .gen = 2, .num_pipes = 1,
         .has_overlay = 1, .overlay_needs_physical = 1,
         .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_i915g_info = {
         .gen = 3, .is_i915g = 1, .cursor_needs_physical = 1, .num_pipes = 2,
         .has_overlay = 1, .overlay_needs_physical = 1,
         .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 static const struct intel_device_info intel_i915gm_info = {
         .gen = 3, .is_mobile = 1, .num_pipes = 2,
@@ -194,11 +93,13 @@ static const struct intel_device_info intel_i915gm_info = {
         .supports_tv = 1,
         .has_fbc = 1,
         .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 static const struct intel_device_info intel_i945g_info = {
         .gen = 3, .has_hotplug = 1, .cursor_needs_physical = 1, .num_pipes = 2,
         .has_overlay = 1, .overlay_needs_physical = 1,
         .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 static const struct intel_device_info intel_i945gm_info = {
         .gen = 3, .is_i945gm = 1, .is_mobile = 1, .num_pipes = 2,
@@ -207,6 +108,7 @@ static const struct intel_device_info intel_i945gm_info = {
         .supports_tv = 1,
         .has_fbc = 1,
         .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_i965g_info = {
@@ -214,6 +116,7 @@ static const struct intel_device_info intel_i965g_info = {
         .has_hotplug = 1,
         .has_overlay = 1,
         .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_i965gm_info = {
@@ -222,6 +125,7 @@ static const struct intel_device_info intel_i965gm_info = {
         .has_overlay = 1,
         .supports_tv = 1,
         .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_g33_info = {
@@ -229,12 +133,14 @@ static const struct intel_device_info intel_g33_info = {
         .need_gfx_hws = 1, .has_hotplug = 1,
         .has_overlay = 1,
         .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_g45_info = {
         .gen = 4, .is_g4x = 1, .need_gfx_hws = 1, .num_pipes = 2,
         .has_pipe_cxsr = 1, .has_hotplug = 1,
         .ring_mask = RENDER_RING | BSD_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_gm45_info = {
@@ -243,18 +149,21 @@ static const struct intel_device_info intel_gm45_info = {
         .has_pipe_cxsr = 1, .has_hotplug = 1,
         .supports_tv = 1,
         .ring_mask = RENDER_RING | BSD_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_pineview_info = {
         .gen = 3, .is_g33 = 1, .is_pineview = 1, .is_mobile = 1, .num_pipes = 2,
         .need_gfx_hws = 1, .has_hotplug = 1,
         .has_overlay = 1,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_ironlake_d_info = {
         .gen = 5, .num_pipes = 2,
         .need_gfx_hws = 1, .has_hotplug = 1,
         .ring_mask = RENDER_RING | BSD_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_ironlake_m_info = {
@@ -262,6 +171,7 @@ static const struct intel_device_info intel_ironlake_m_info = {
         .need_gfx_hws = 1, .has_hotplug = 1,
         .has_fbc = 1,
         .ring_mask = RENDER_RING | BSD_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_sandybridge_d_info = {
@@ -270,6 +180,7 @@ static const struct intel_device_info intel_sandybridge_d_info = {
         .has_fbc = 1,
         .ring_mask = RENDER_RING | BSD_RING | BLT_RING,
         .has_llc = 1,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_sandybridge_m_info = {
@@ -278,6 +189,7 @@ static const struct intel_device_info intel_sandybridge_m_info = {
         .has_fbc = 1,
         .ring_mask = RENDER_RING | BSD_RING | BLT_RING,
         .has_llc = 1,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 #define GEN7_FEATURES  \
@@ -290,18 +202,21 @@ static const struct intel_device_info intel_sandybridge_m_info = {
 static const struct intel_device_info intel_ivybridge_d_info = {
         GEN7_FEATURES,
         .is_ivybridge = 1,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_ivybridge_m_info = {
         GEN7_FEATURES,
         .is_ivybridge = 1,
         .is_mobile = 1,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_ivybridge_q_info = {
         GEN7_FEATURES,
         .is_ivybridge = 1,
         .num_pipes = 0, /* legal, last one wins */
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_valleyview_m_info = {
@@ -312,6 +227,7 @@ static const struct intel_device_info intel_valleyview_m_info = {
         .display_mmio_offset = VLV_DISPLAY_BASE,
         .has_fbc = 0, /* legal, last one wins */
         .has_llc = 0, /* legal, last one wins */
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_valleyview_d_info = {
@@ -321,6 +237,7 @@ static const struct intel_device_info intel_valleyview_d_info = {
         .display_mmio_offset = VLV_DISPLAY_BASE,
         .has_fbc = 0, /* legal, last one wins */
         .has_llc = 0, /* legal, last one wins */
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_haswell_d_info = {
@@ -329,6 +246,7 @@ static const struct intel_device_info intel_haswell_d_info = {
         .has_ddi = 1,
         .has_fpga_dbg = 1,
         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_haswell_m_info = {
@@ -338,6 +256,7 @@ static const struct intel_device_info intel_haswell_m_info = {
         .has_ddi = 1,
         .has_fpga_dbg = 1,
         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_broadwell_d_info = {
@@ -346,6 +265,8 @@ static const struct intel_device_info intel_broadwell_d_info = {
         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
         .has_llc = 1,
         .has_ddi = 1,
+        .has_fbc = 1,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 static const struct intel_device_info intel_broadwell_m_info = {
@@ -354,6 +275,8 @@ static const struct intel_device_info intel_broadwell_m_info = {
         .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
         .has_llc = 1,
         .has_ddi = 1,
+        .has_fbc = 1,
+        GEN_DEFAULT_PIPEOFFSETS,
 };
 
 /*
@@ -475,14 +398,12 @@ bool i915_semaphore_is_enabled(struct drm_device *dev)
         if (INTEL_INFO(dev)->gen < 6)
                 return false;
 
+        if (i915.semaphores >= 0)
+                return i915.semaphores;
+
         /* Until we get further testing... */
-        if (IS_GEN8(dev)) {
-                WARN_ON(!i915_preliminary_hw_support);
+        if (IS_GEN8(dev))
                 return false;
-        }
-
-        if (i915_semaphores >= 0)
-                return i915_semaphores;
 
 #ifdef CONFIG_INTEL_IOMMU
         /* Enable semaphores on SNB when IO remapping is off */
@@ -507,8 +428,7 @@ static int i915_drm_freeze(struct drm_device *dev)
 
         /* We do a lot of poking in a lot of registers, make sure they work
          * properly. */
-        hsw_disable_package_c8(dev_priv);
-        intel_display_set_init_power(dev, true);
+        intel_display_set_init_power(dev_priv, true);
 
         drm_kms_helper_poll_disable(dev);
 
@@ -546,11 +466,14 @@ static int i915_drm_freeze(struct drm_device *dev)
         i915_save_state(dev);
 
         intel_opregion_fini(dev);
+        intel_uncore_fini(dev);
 
         console_lock();
         intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED);
         console_unlock();
 
+        dev_priv->suspend_count++;
+
         return 0;
 }
 
@@ -614,14 +537,21 @@ static void intel_resume_hotplug(struct drm_device *dev)
         drm_helper_hpd_irq_event(dev);
 }
 
-static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings)
+static int i915_drm_thaw_early(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int error = 0;
 
         intel_uncore_early_sanitize(dev);
-
         intel_uncore_sanitize(dev);
+        intel_power_domains_init_hw(dev_priv);
+
+        return 0;
+}
+
+static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int error = 0;
 
         if (drm_core_check_feature(dev, DRIVER_MODESET) &&
             restore_gtt_mappings) {
@@ -630,14 +560,13 @@ static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings)
                 mutex_unlock(&dev->struct_mutex);
         }
 
-        intel_power_domains_init_hw(dev);
-
         i915_restore_state(dev);
         intel_opregion_setup(dev);
 
         /* KMS EnterVT equivalent */
         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
                 intel_init_pch_refclk(dev);
+                drm_mode_config_reset(dev);
 
                 mutex_lock(&dev->struct_mutex);
 
@@ -650,7 +579,6 @@ static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings)
                 intel_modeset_init_hw(dev);
 
                 drm_modeset_lock_all(dev);
-                drm_mode_config_reset(dev);
                 intel_modeset_setup_hw_state(dev, true);
                 drm_modeset_unlock_all(dev);
 
@@ -680,10 +608,6 @@ static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings)
                 schedule_work(&dev_priv->console_resume_work);
         }
 
-        /* Undo what we did at i915_drm_freeze so the refcount goes back to the
-         * expected level. */
-        hsw_enable_package_c8(dev_priv);
-
         mutex_lock(&dev_priv->modeset_restore_lock);
         dev_priv->modeset_restore = MODESET_DONE;
         mutex_unlock(&dev_priv->modeset_restore_lock);
@@ -700,19 +624,33 @@ static int i915_drm_thaw(struct drm_device *dev)
         return __i915_drm_thaw(dev, true);
 }
 
-int i915_resume(struct drm_device *dev)
+static int i915_resume_early(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int ret;
-
         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
                 return 0;
 
+        /*
+         * We have a resume ordering issue with the snd-hda driver also
+         * requiring our device to be power up. Due to the lack of a
+         * parent/child relationship we currently solve this with an early
+         * resume hook.
+         *
+         * FIXME: This should be solved with a special hdmi sink device or
+         * similar so that power domains can be employed.
+         */
         if (pci_enable_device(dev->pdev))
                 return -EIO;
 
         pci_set_master(dev->pdev);
 
+        return i915_drm_thaw_early(dev);
+}
+
+int i915_resume(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
+
         /*
          * Platforms with opregion should have sane BIOS, older ones (gen3 and
          * earlier) need to restore the GTT mappings since the BIOS might clear
@@ -726,6 +664,14 @@ int i915_resume(struct drm_device *dev)
         return 0;
 }
 
+static int i915_resume_legacy(struct drm_device *dev)
+{
+        i915_resume_early(dev);
+        i915_resume(dev);
+
+        return 0;
+}
+
 /**
  * i915_reset - reset chip after a hang
  * @dev: drm device to reset
@@ -743,11 +689,11 @@ int i915_resume(struct drm_device *dev)
  */
 int i915_reset(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         bool simulated;
         int ret;
 
-        if (!i915_try_reset)
+        if (!i915.reset)
                 return 0;
 
         mutex_lock(&dev->struct_mutex);
@@ -802,6 +748,17 @@ int i915_reset(struct drm_device *dev)
 
                 drm_irq_uninstall(dev);
                 drm_irq_install(dev);
+
+                /* rps/rc6 re-init is necessary to restore state lost after the
+                 * reset and the re-install of drm irq. Skip for ironlake per
+                 * previous concerns that it doesn't respond well to some forms
+                 * of re-init after reset. */
+                if (INTEL_INFO(dev)->gen > 5) {
+                        mutex_lock(&dev->struct_mutex);
+                        intel_enable_gt_powersave(dev);
+                        mutex_unlock(&dev->struct_mutex);
+                }
+
                 intel_hpd_init(dev);
         } else {
                 mutex_unlock(&dev->struct_mutex);
@@ -815,7 +772,7 @@ static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
         struct intel_device_info *intel_info =
                 (struct intel_device_info *) ent->driver_data;
 
-        if (IS_PRELIMINARY_HW(intel_info) && !i915_preliminary_hw_support) {
+        if (IS_PRELIMINARY_HW(intel_info) && !i915.preliminary_hw_support) {
                 DRM_INFO("This hardware requires preliminary hardware support.\n"
                          "See CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT, and/or modparam preliminary_hw_support\n");
                 return -ENODEV;
@@ -846,7 +803,6 @@ static int i915_pm_suspend(struct device *dev)
 {
         struct pci_dev *pdev = to_pci_dev(dev);
         struct drm_device *drm_dev = pci_get_drvdata(pdev);
-        int error;
 
         if (!drm_dev || !drm_dev->dev_private) {
                 dev_err(dev, "DRM not initialized, aborting suspend.\n");
@@ -856,9 +812,25 @@ static int i915_pm_suspend(struct device *dev)
         if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
                 return 0;
 
-        error = i915_drm_freeze(drm_dev);
-        if (error)
-                return error;
+        return i915_drm_freeze(drm_dev);
+}
+
+static int i915_pm_suspend_late(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+
+        /*
+         * We have a suspedn ordering issue with the snd-hda driver also
+         * requiring our device to be power up. Due to the lack of a
+         * parent/child relationship we currently solve this with an late
+         * suspend hook.
+         *
+         * FIXME: This should be solved with a special hdmi sink device or
+         * similar so that power domains can be employed.
+         */
+        if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
+                return 0;
 
         pci_disable_device(pdev);
         pci_set_power_state(pdev, PCI_D3hot);
@@ -866,6 +838,14 @@ static int i915_pm_suspend(struct device *dev)
         return 0;
 }
 
+static int i915_pm_resume_early(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+
+        return i915_resume_early(drm_dev);
+}
+
 static int i915_pm_resume(struct device *dev)
 {
         struct pci_dev *pdev = to_pci_dev(dev);
@@ -887,6 +867,14 @@ static int i915_pm_freeze(struct device *dev)
         return i915_drm_freeze(drm_dev);
 }
 
+static int i915_pm_thaw_early(struct device *dev)
+{
+        struct pci_dev *pdev = to_pci_dev(dev);
+        struct drm_device *drm_dev = pci_get_drvdata(pdev);
+
+        return i915_drm_thaw_early(drm_dev);
+}
+
 static int i915_pm_thaw(struct device *dev)
 {
         struct pci_dev *pdev = to_pci_dev(dev);
@@ -910,9 +898,13 @@ static int i915_runtime_suspend(struct device *device)
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         WARN_ON(!HAS_RUNTIME_PM(dev));
+        assert_force_wake_inactive(dev_priv);
 
         DRM_DEBUG_KMS("Suspending device\n");
 
+        if (HAS_PC8(dev))
+                hsw_enable_pc8(dev_priv);
+
         i915_gem_release_all_mmaps(dev_priv);
 
         del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
@@ -927,6 +919,7 @@ static int i915_runtime_suspend(struct device *device)
          */
         intel_opregion_notify_adapter(dev, PCI_D1);
 
+        DRM_DEBUG_KMS("Device suspended\n");
         return 0;
 }
 
@@ -943,15 +936,23 @@ static int i915_runtime_resume(struct device *device)
         intel_opregion_notify_adapter(dev, PCI_D0);
         dev_priv->pm.suspended = false;
 
+        if (HAS_PC8(dev))
+                hsw_disable_pc8(dev_priv);
+
+        DRM_DEBUG_KMS("Device resumed\n");
         return 0;
 }
 
 static const struct dev_pm_ops i915_pm_ops = {
         .suspend = i915_pm_suspend,
+        .suspend_late = i915_pm_suspend_late,
+        .resume_early = i915_pm_resume_early,
         .resume = i915_pm_resume,
         .freeze = i915_pm_freeze,
+        .thaw_early = i915_pm_thaw_early,
         .thaw = i915_pm_thaw,
         .poweroff = i915_pm_poweroff,
+        .restore_early = i915_pm_resume_early,
         .restore = i915_pm_resume,
         .runtime_suspend = i915_runtime_suspend,
         .runtime_resume = i915_runtime_resume,
@@ -994,7 +995,7 @@ static struct drm_driver driver = {
 
         /* Used in place of i915_pm_ops for non-DRIVER_MODESET */
         .suspend = i915_suspend,
-        .resume = i915_resume,
+        .resume = i915_resume_legacy,
 
         .device_is_agp = i915_driver_device_is_agp,
         .master_create = i915_master_create,
@@ -1046,14 +1047,14 @@ static int __init i915_init(void)
          * the default behavior.
          */
 #if defined(CONFIG_DRM_I915_KMS)
-        if (i915_modeset != 0)
+        if (i915.modeset != 0)
                 driver.driver_features |= DRIVER_MODESET;
 #endif
-        if (i915_modeset == 1)
+        if (i915.modeset == 1)
                 driver.driver_features |= DRIVER_MODESET;
 
 #ifdef CONFIG_VGA_CONSOLE
-        if (vgacon_text_force() && i915_modeset == -1)
+        if (vgacon_text_force() && i915.modeset == -1)
                 driver.driver_features &= ~DRIVER_MODESET;
 #endif
 
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index df77e20e3c3d..0905cd915589 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -58,7 +58,8 @@ enum pipe {
         PIPE_A = 0,
         PIPE_B,
         PIPE_C,
-        I915_MAX_PIPES
+        _PIPE_EDP,
+        I915_MAX_PIPES = _PIPE_EDP
 };
 #define pipe_name(p) ((p) + 'A')
 
@@ -66,7 +67,8 @@ enum transcoder {
         TRANSCODER_A = 0,
         TRANSCODER_B,
         TRANSCODER_C,
-        TRANSCODER_EDP = 0xF,
+        TRANSCODER_EDP,
+        I915_MAX_TRANSCODERS
 };
 #define transcoder_name(t) ((t) + 'A')
 
@@ -77,7 +79,7 @@ enum plane {
 };
 #define plane_name(p) ((p) + 'A')
 
-#define sprite_name(p, s) ((p) * dev_priv->num_plane + (s) + 'A')
+#define sprite_name(p, s) ((p) * INTEL_INFO(dev)->num_sprites[(p)] + (s) + 'A')
 
 enum port {
         PORT_A = 0,
@@ -112,6 +114,17 @@ enum intel_display_power_domain {
         POWER_DOMAIN_TRANSCODER_B,
         POWER_DOMAIN_TRANSCODER_C,
         POWER_DOMAIN_TRANSCODER_EDP,
+        POWER_DOMAIN_PORT_DDI_A_2_LANES,
+        POWER_DOMAIN_PORT_DDI_A_4_LANES,
+        POWER_DOMAIN_PORT_DDI_B_2_LANES,
+        POWER_DOMAIN_PORT_DDI_B_4_LANES,
+        POWER_DOMAIN_PORT_DDI_C_2_LANES,
+        POWER_DOMAIN_PORT_DDI_C_4_LANES,
+        POWER_DOMAIN_PORT_DDI_D_2_LANES,
+        POWER_DOMAIN_PORT_DDI_D_4_LANES,
+        POWER_DOMAIN_PORT_DSI,
+        POWER_DOMAIN_PORT_CRT,
+        POWER_DOMAIN_PORT_OTHER,
         POWER_DOMAIN_VGA,
         POWER_DOMAIN_AUDIO,
         POWER_DOMAIN_INIT,
@@ -119,8 +132,6 @@ enum intel_display_power_domain {
         POWER_DOMAIN_NUM,
 };
 
-#define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1)
-
 #define POWER_DOMAIN_PIPE(pipe) ((pipe) + POWER_DOMAIN_PIPE_A)
 #define POWER_DOMAIN_PIPE_PANEL_FITTER(pipe) \
                 ((pipe) + POWER_DOMAIN_PIPE_A_PANEL_FITTER)
@@ -128,14 +139,6 @@ enum intel_display_power_domain {
         ((tran) == TRANSCODER_EDP ? POWER_DOMAIN_TRANSCODER_EDP : \
          (tran) + POWER_DOMAIN_TRANSCODER_A)
 
-#define HSW_ALWAYS_ON_POWER_DOMAINS (           \
-        BIT(POWER_DOMAIN_PIPE_A) |              \
-        BIT(POWER_DOMAIN_TRANSCODER_EDP))
-#define BDW_ALWAYS_ON_POWER_DOMAINS (           \
-        BIT(POWER_DOMAIN_PIPE_A) |              \
-        BIT(POWER_DOMAIN_TRANSCODER_EDP) |      \
-        BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER))
-
 enum hpd_pin {
         HPD_NONE = 0,
         HPD_PORT_A = HPD_NONE, /* PORT_A is internal */
@@ -157,11 +160,16 @@ enum hpd_pin {
          I915_GEM_DOMAIN_VERTEX)
 
 #define for_each_pipe(p) for ((p) = 0; (p) < INTEL_INFO(dev)->num_pipes; (p)++)
+#define for_each_sprite(p, s) for ((s) = 0; (s) < INTEL_INFO(dev)->num_sprites[(p)]; (s)++)
 
 #define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \
         list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \
                 if ((intel_encoder)->base.crtc == (__crtc))
 
+#define for_each_connector_on_encoder(dev, __encoder, intel_connector) \
+        list_for_each_entry((intel_connector), &(dev)->mode_config.connector_list, base.head) \
+                if ((intel_connector)->base.encoder == (__encoder))
+
 struct drm_i915_private;
 
 enum intel_dpll_id {
@@ -295,53 +303,87 @@ struct intel_display_error_state;
 
 struct drm_i915_error_state {
         struct kref ref;
+        struct timeval time;
+
+        char error_msg[128];
+        u32 reset_count;
+        u32 suspend_count;
+
+        /* Generic register state */
         u32 eir;
         u32 pgtbl_er;
         u32 ier;
         u32 ccid;
         u32 derrmr;
         u32 forcewake;
-        bool waiting[I915_NUM_RINGS];
-        u32 pipestat[I915_MAX_PIPES];
-        u32 tail[I915_NUM_RINGS];
-        u32 head[I915_NUM_RINGS];
-        u32 ctl[I915_NUM_RINGS];
-        u32 ipeir[I915_NUM_RINGS];
-        u32 ipehr[I915_NUM_RINGS];
-        u32 instdone[I915_NUM_RINGS];
-        u32 acthd[I915_NUM_RINGS];
-        u32 semaphore_mboxes[I915_NUM_RINGS][I915_NUM_RINGS - 1];
-        u32 semaphore_seqno[I915_NUM_RINGS][I915_NUM_RINGS - 1];
-        u32 rc_psmi[I915_NUM_RINGS]; /* sleep state */
-        /* our own tracking of ring head and tail */
-        u32 cpu_ring_head[I915_NUM_RINGS];
-        u32 cpu_ring_tail[I915_NUM_RINGS];
         u32 error; /* gen6+ */
         u32 err_int; /* gen7 */
-        u32 bbstate[I915_NUM_RINGS];
-        u32 instpm[I915_NUM_RINGS];
-        u32 instps[I915_NUM_RINGS];
-        u32 extra_instdone[I915_NUM_INSTDONE_REG];
-        u32 seqno[I915_NUM_RINGS];
-        u64 bbaddr[I915_NUM_RINGS];
-        u32 fault_reg[I915_NUM_RINGS];
         u32 done_reg;
-        u32 faddr[I915_NUM_RINGS];
+        u32 gac_eco;
+        u32 gam_ecochk;
+        u32 gab_ctl;
+        u32 gfx_mode;
+        u32 extra_instdone[I915_NUM_INSTDONE_REG];
+        u32 pipestat[I915_MAX_PIPES];
         u64 fence[I915_MAX_NUM_FENCES];
-        struct timeval time;
+        struct intel_overlay_error_state *overlay;
+        struct intel_display_error_state *display;
+
         struct drm_i915_error_ring {
                 bool valid;
+                /* Software tracked state */
+                bool waiting;
+                int hangcheck_score;
+                enum intel_ring_hangcheck_action hangcheck_action;
+                int num_requests;
+
+                /* our own tracking of ring head and tail */
+                u32 cpu_ring_head;
+                u32 cpu_ring_tail;
+
+                u32 semaphore_seqno[I915_NUM_RINGS - 1];
+
+                /* Register state */
+                u32 tail;
+                u32 head;
+                u32 ctl;
+                u32 hws;
+                u32 ipeir;
+                u32 ipehr;
+                u32 instdone;
+                u32 bbstate;
+                u32 instpm;
+                u32 instps;
+                u32 seqno;
+                u64 bbaddr;
+                u64 acthd;
+                u32 fault_reg;
+                u32 faddr;
+                u32 rc_psmi; /* sleep state */
+                u32 semaphore_mboxes[I915_NUM_RINGS - 1];
+
                 struct drm_i915_error_object {
                         int page_count;
                         u32 gtt_offset;
                         u32 *pages[0];
-                } *ringbuffer, *batchbuffer, *ctx;
+                } *ringbuffer, *batchbuffer, *wa_batchbuffer, *ctx, *hws_page;
+
                 struct drm_i915_error_request {
                         long jiffies;
                         u32 seqno;
                         u32 tail;
                 } *requests;
-                int num_requests;
+
+                struct {
+                        u32 gfx_mode;
+                        union {
+                                u64 pdp[4];
+                                u32 pp_dir_base;
+                        };
+                } vm_info;
+
+                pid_t pid;
+                char comm[TASK_COMM_LEN];
         } ring[I915_NUM_RINGS];
         struct drm_i915_error_buffer {
                 u32 size;
@@ -358,15 +400,13 @@ struct drm_i915_error_state {
                 s32 ring:4;
                 u32 cache_level:3;
         } **active_bo, **pinned_bo;
+
         u32 *active_bo_count, *pinned_bo_count;
-        struct intel_overlay_error_state *overlay;
-        struct intel_display_error_state *display;
-        int hangcheck_score[I915_NUM_RINGS];
-        enum intel_ring_hangcheck_action hangcheck_action[I915_NUM_RINGS];
 };
 
 struct intel_connector;
 struct intel_crtc_config;
+struct intel_plane_config;
 struct intel_crtc;
 struct intel_limit;
 struct dpll;
@@ -405,6 +445,8 @@ struct drm_i915_display_funcs {
          * fills out the pipe-config with the hw state. */
         bool (*get_pipe_config)(struct intel_crtc *,
                                 struct intel_crtc_config *);
+        void (*get_plane_config)(struct intel_crtc *,
+                                 struct intel_plane_config *);
         int (*crtc_mode_set)(struct drm_crtc *crtc,
                              int x, int y,
                              struct drm_framebuffer *old_fb);
@@ -420,8 +462,9 @@ struct drm_i915_display_funcs {
                           struct drm_framebuffer *fb,
                           struct drm_i915_gem_object *obj,
                           uint32_t flags);
-        int (*update_plane)(struct drm_crtc *crtc, struct drm_framebuffer *fb,
-                            int x, int y);
+        int (*update_primary_plane)(struct drm_crtc *crtc,
+                                    struct drm_framebuffer *fb,
+                                    int x, int y);
         void (*hpd_irq_setup)(struct drm_device *dev);
         /* clock updates for mode set */
         /* cursor updates */
@@ -469,7 +512,7 @@ struct intel_uncore {
         unsigned fw_rendercount;
         unsigned fw_mediacount;
 
-        struct delayed_work force_wake_work;
+        struct timer_list force_wake_timer;
 };
 
 #define DEV_INFO_FOR_EACH_FLAG(func, sep) \
@@ -504,9 +547,16 @@ struct intel_uncore {
 struct intel_device_info {
         u32 display_mmio_offset;
         u8 num_pipes:3;
+        u8 num_sprites[I915_MAX_PIPES];
         u8 gen;
         u8 ring_mask; /* Rings supported by the HW */
         DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG, SEP_SEMICOLON);
+        /* Register offsets for the various display pipes and transcoders */
+        int pipe_offsets[I915_MAX_TRANSCODERS];
+        int trans_offsets[I915_MAX_TRANSCODERS];
+        int dpll_offsets[I915_MAX_PIPES];
+        int dpll_md_offsets[I915_MAX_PIPES];
+        int palette_offsets[I915_MAX_PIPES];
 };
 
 #undef DEFINE_FLAG
@@ -524,6 +574,57 @@ enum i915_cache_level {
 
 typedef uint32_t gen6_gtt_pte_t;
 
+/**
+ * A VMA represents a GEM BO that is bound into an address space. Therefore, a
+ * VMA's presence cannot be guaranteed before binding, or after unbinding the
+ * object into/from the address space.
+ *
+ * To make things as simple as possible (ie. no refcounting), a VMA's lifetime
+ * will always be <= an objects lifetime. So object refcounting should cover us.
+ */
+struct i915_vma {
+        struct drm_mm_node node;
+        struct drm_i915_gem_object *obj;
+        struct i915_address_space *vm;
+
+        /** This object's place on the active/inactive lists */
+        struct list_head mm_list;
+
+        struct list_head vma_link; /* Link in the object's VMA list */
+
+        /** This vma's place in the batchbuffer or on the eviction list */
+        struct list_head exec_list;
+
+        /**
+         * Used for performing relocations during execbuffer insertion.
+         */
+        struct hlist_node exec_node;
+        unsigned long exec_handle;
+        struct drm_i915_gem_exec_object2 *exec_entry;
+
+        /**
+         * How many users have pinned this object in GTT space. The following
+         * users can each hold at most one reference: pwrite/pread, pin_ioctl
+         * (via user_pin_count), execbuffer (objects are not allowed multiple
+         * times for the same batchbuffer), and the framebuffer code. When
+         * switching/pageflipping, the framebuffer code has at most two buffers
+         * pinned per crtc.
+         *
+         * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
+         * bits with absolutely no headroom. So use 4 bits. */
+        unsigned int pin_count:4;
+#define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf
+
+        /** Unmap an object from an address space. This usually consists of
+         * setting the valid PTE entries to a reserved scratch page. */
+        void (*unbind_vma)(struct i915_vma *vma);
+        /* Map an object into an address space with the given cache flags. */
+#define GLOBAL_BIND (1<<0)
+        void (*bind_vma)(struct i915_vma *vma,
+                         enum i915_cache_level cache_level,
+                         u32 flags);
+};
+
 struct i915_address_space {
         struct drm_mm mm;
         struct drm_device *dev;
@@ -564,12 +665,12 @@ struct i915_address_space {
                                      enum i915_cache_level level,
                                      bool valid); /* Create a valid PTE */
         void (*clear_range)(struct i915_address_space *vm,
-                            unsigned int first_entry,
-                            unsigned int num_entries,
+                            uint64_t start,
+                            uint64_t length,
                             bool use_scratch);
         void (*insert_entries)(struct i915_address_space *vm,
                                struct sg_table *st,
-                               unsigned int first_entry,
+                               uint64_t start,
                                enum i915_cache_level cache_level);
         void (*cleanup)(struct i915_address_space *vm);
 };
@@ -603,55 +704,34 @@ struct i915_gtt {
 };
 #define gtt_total_entries(gtt) ((gtt).base.total >> PAGE_SHIFT)
 
+#define GEN8_LEGACY_PDPS 4
 struct i915_hw_ppgtt {
         struct i915_address_space base;
+        struct kref ref;
+        struct drm_mm_node node;
         unsigned num_pd_entries;
+        unsigned num_pd_pages; /* gen8+ */
         union {
                 struct page **pt_pages;
-                struct page *gen8_pt_pages;
+                struct page **gen8_pt_pages[GEN8_LEGACY_PDPS];
         };
         struct page *pd_pages;
-        int num_pd_pages;
-        int num_pt_pages;
         union {
                 uint32_t pd_offset;
-                dma_addr_t pd_dma_addr[4];
+                dma_addr_t pd_dma_addr[GEN8_LEGACY_PDPS];
         };
         union {
                 dma_addr_t *pt_dma_addr;
                 dma_addr_t *gen8_pt_dma_addr[4];
         };
-        int (*enable)(struct drm_device *dev);
-};
-
-/**
- * A VMA represents a GEM BO that is bound into an address space. Therefore, a
- * VMA's presence cannot be guaranteed before binding, or after unbinding the
- * object into/from the address space.
- *
- * To make things as simple as possible (ie. no refcounting), a VMA's lifetime
- * will always be <= an objects lifetime. So object refcounting should cover us.
- */
-struct i915_vma {
-        struct drm_mm_node node;
-        struct drm_i915_gem_object *obj;
-        struct i915_address_space *vm;
-
-        /** This object's place on the active/inactive lists */
-        struct list_head mm_list;
 
-        struct list_head vma_link; /* Link in the object's VMA list */
-
-        /** This vma's place in the batchbuffer or on the eviction list */
-        struct list_head exec_list;
-
-        /**
-         * Used for performing relocations during execbuffer insertion.
-         */
-        struct hlist_node exec_node;
-        unsigned long exec_handle;
-        struct drm_i915_gem_exec_object2 *exec_entry;
+        struct i915_hw_context *ctx;
 
+        int (*enable)(struct i915_hw_ppgtt *ppgtt);
+        int (*switch_mm)(struct i915_hw_ppgtt *ppgtt,
+                         struct intel_ring_buffer *ring,
+                         bool synchronous);
+        void (*debug_dump)(struct i915_hw_ppgtt *ppgtt, struct seq_file *m);
 };
 
 struct i915_ctx_hang_stats {
@@ -676,9 +756,10 @@ struct i915_hw_context {
         bool is_initialized;
         uint8_t remap_slice;
         struct drm_i915_file_private *file_priv;
-        struct intel_ring_buffer *ring;
+        struct intel_ring_buffer *last_ring;
         struct drm_i915_gem_object *obj;
         struct i915_ctx_hang_stats hang_stats;
+        struct i915_address_space *vm;
 
         struct list_head link;
 };
@@ -831,11 +912,7 @@ struct i915_suspend_saved_registers {
         u32 savePFIT_CONTROL;
         u32 save_palette_a[256];
         u32 save_palette_b[256];
-        u32 saveDPFC_CB_BASE;
-        u32 saveFBC_CFB_BASE;
-        u32 saveFBC_LL_BASE;
         u32 saveFBC_CONTROL;
-        u32 saveFBC_CONTROL2;
         u32 saveIER;
         u32 saveIIR;
         u32 saveIMR;
@@ -905,15 +982,24 @@ struct intel_gen6_power_mgmt {
         struct work_struct work;
         u32 pm_iir;
 
-        /* The below variables an all the rps hw state are protected by
-         * dev->struct mutext. */
-        u8 cur_delay;
-        u8 min_delay;
-        u8 max_delay;
-        u8 rpe_delay;
-        u8 rp1_delay;
-        u8 rp0_delay;
-        u8 hw_max;
+        /* Frequencies are stored in potentially platform dependent multiples.
+         * In other words, *_freq needs to be multiplied by X to be interesting.
+         * Soft limits are those which are used for the dynamic reclocking done
+         * by the driver (raise frequencies under heavy loads, and lower for
+         * lighter loads). Hard limits are those imposed by the hardware.
+         *
+         * A distinction is made for overclocking, which is never enabled by
+         * default, and is considered to be above the hard limit if it's
+         * possible at all.
+         */
+        u8 cur_freq;            /* Current frequency (cached, may not == HW) */
+        u8 min_freq_softlimit;  /* Minimum frequency permitted by the driver */
+        u8 max_freq_softlimit;  /* Max frequency permitted by the driver */
+        u8 max_freq;            /* Maximum frequency, RP0 if not overclocking */
+        u8 min_freq;            /* AKA RPn. Minimum frequency */
+        u8 efficient_freq;      /* AKA RPe. Pre-determined balanced frequency */
+        u8 rp1_freq;            /* "less than" RP0 power/freqency */
+        u8 rp0_freq;            /* Non-overclocked max frequency. */
 
         int last_adj;
         enum { LOW_POWER, BETWEEN, HIGH_POWER } power;
@@ -953,6 +1039,36 @@ struct intel_ilk_power_mgmt {
         struct drm_i915_gem_object *renderctx;
 };
 
+struct drm_i915_private;
+struct i915_power_well;
+
+struct i915_power_well_ops {
+        /*
+         * Synchronize the well's hw state to match the current sw state, for
+         * example enable/disable it based on the current refcount. Called
+         * during driver init and resume time, possibly after first calling
+         * the enable/disable handlers.
+         */
+        void (*sync_hw)(struct drm_i915_private *dev_priv,
+                        struct i915_power_well *power_well);
+        /*
+         * Enable the well and resources that depend on it (for example
+         * interrupts located on the well). Called after the 0->1 refcount
+         * transition.
+         */
+        void (*enable)(struct drm_i915_private *dev_priv,
+                       struct i915_power_well *power_well);
+        /*
+         * Disable the well and resources that depend on it. Called after
+         * the 1->0 refcount transition.
+         */
+        void (*disable)(struct drm_i915_private *dev_priv,
+                        struct i915_power_well *power_well);
+        /* Returns the hw enabled state. */
+        bool (*is_enabled)(struct drm_i915_private *dev_priv,
+                           struct i915_power_well *power_well);
+};
+
 /* Power well structure for haswell */
 struct i915_power_well {
         const char *name;
@@ -960,11 +1076,8 @@ struct i915_power_well {
         /* power well enable/disable usage count */
         int count;
         unsigned long domains;
-        void *data;
-        void (*set)(struct drm_device *dev, struct i915_power_well *power_well,
-                    bool enable);
-        bool (*is_enabled)(struct drm_device *dev,
-                           struct i915_power_well *power_well);
+        unsigned long data;
+        const struct i915_power_well_ops *ops;
 };
 
 struct i915_power_domains {
@@ -1061,6 +1174,14 @@ struct i915_gem_mm {
          */
         bool interruptible;
 
+        /**
+         * Is the GPU currently considered idle, or busy executing userspace
+         * requests?  Whilst idle, we attempt to power down the hardware and
+         * display clocks. In order to reduce the effect on performance, there
+         * is a slight delay before we do so.
+         */
+        bool busy;
+
         /** Bit 6 swizzling required for X tiling */
         uint32_t bit_6_swizzle_x;
         /** Bit 6 swizzling required for Y tiling */
@@ -1226,44 +1347,19 @@ struct ilk_wm_values {
 };
 
 /*
- * This struct tracks the state needed for the Package C8+ feature.
- *
- * Package states C8 and deeper are really deep PC states that can only be
- * reached when all the devices on the system allow it, so even if the graphics
- * device allows PC8+, it doesn't mean the system will actually get to these
- * states.
- *
- * Our driver only allows PC8+ when all the outputs are disabled, the power well
- * is disabled and the GPU is idle. When these conditions are met, we manually
- * do the other conditions: disable the interrupts, clocks and switch LCPLL
- * refclk to Fclk.
- *
- * When we really reach PC8 or deeper states (not just when we allow it) we lose
- * the state of some registers, so when we come back from PC8+ we need to
- * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
- * need to take care of the registers kept by RC6.
+ * This struct helps tracking the state needed for runtime PM, which puts the
+ * device in PCI D3 state. Notice that when this happens, nothing on the
+ * graphics device works, even register access, so we don't get interrupts nor
+ * anything else.
  *
- * The interrupt disabling is part of the requirements. We can only leave the
- * PCH HPD interrupts enabled. If we're in PC8+ and we get another interrupt we
- * can lock the machine.
+ * Every piece of our code that needs to actually touch the hardware needs to
+ * either call intel_runtime_pm_get or call intel_display_power_get with the
+ * appropriate power domain.
  *
- * Ideally every piece of our code that needs PC8+ disabled would call
- * hsw_disable_package_c8, which would increment disable_count and prevent the
- * system from reaching PC8+. But we don't have a symmetric way to do this for
- * everything, so we have the requirements_met and gpu_idle variables. When we
- * switch requirements_met or gpu_idle to true we decrease disable_count, and
- * increase it in the opposite case. The requirements_met variable is true when
- * all the CRTCs, encoders and the power well are disabled. The gpu_idle
- * variable is true when the GPU is idle.
- *
- * In addition to everything, we only actually enable PC8+ if disable_count
- * stays at zero for at least some seconds. This is implemented with the
- * enable_work variable. We do this so we don't enable/disable PC8 dozens of
- * consecutive times when all screens are disabled and some background app
- * queries the state of our connectors, or we have some application constantly
- * waking up to use the GPU. Only after the enable_work function actually
- * enables PC8+ the "enable" variable will become true, which means that it can
- * be false even if disable_count is 0.
+ * Our driver uses the autosuspend delay feature, which means we'll only really
+ * suspend if we stay with zero refcount for a certain amount of time. The
+ * default value is currently very conservative (see intel_init_runtime_pm), but
+ * it can be changed with the standard runtime PM files from sysfs.
  *
  * The irqs_disabled variable becomes true exactly after we disable the IRQs and
  * goes back to false exactly before we reenable the IRQs. We use this variable
@@ -1273,17 +1369,11 @@ struct ilk_wm_values {
  * inside struct regsave so when we restore the IRQs they will contain the
  * latest expected values.
  *
- * For more, read "Display Sequences for Package C8" on our documentation.
+ * For more, read the Documentation/power/runtime_pm.txt.
  */
-struct i915_package_c8 {
-        bool requirements_met;
-        bool gpu_idle;
+struct i915_runtime_pm {
+        bool suspended;
         bool irqs_disabled;
-        /* Only true after the delayed work task actually enables it. */
-        bool enabled;
-        int disable_count;
-        struct mutex lock;
-        struct delayed_work enable_work;
 
         struct {
                 uint32_t deimr;
@@ -1294,10 +1384,6 @@ struct i915_package_c8 {
         } regsave;
 };
 
-struct i915_runtime_pm {
-        bool suspended;
-};
-
 enum intel_pipe_crc_source {
         INTEL_PIPE_CRC_SOURCE_NONE,
         INTEL_PIPE_CRC_SOURCE_PLANE1,
@@ -1332,7 +1418,7 @@ typedef struct drm_i915_private {
         struct drm_device *dev;
         struct kmem_cache *slab;
 
-        const struct intel_device_info *info;
+        const struct intel_device_info info;
 
         int relative_constants_mode;
 
@@ -1361,11 +1447,11 @@ typedef struct drm_i915_private {
         drm_dma_handle_t *status_page_dmah;
         struct resource mch_res;
 
-        atomic_t irq_received;
-
         /* protects the irq masks */
         spinlock_t irq_lock;
 
+        bool display_irqs_enabled;
+
         /* To control wakeup latency, e.g. for irq-driven dp aux transfers. */
         struct pm_qos_request pm_qos;
 
@@ -1379,6 +1465,8 @@ typedef struct drm_i915_private {
         };
         u32 gt_irq_mask;
         u32 pm_irq_mask;
+        u32 pm_rps_events;
+        u32 pipestat_irq_mask[I915_MAX_PIPES];
 
         struct work_struct hotplug_work;
         bool enable_hotplug_processing;
@@ -1394,8 +1482,6 @@ typedef struct drm_i915_private {
         u32 hpd_event_bits;
         struct timer_list hotplug_reenable_timer;
 
-        int num_plane;
-
         struct i915_fbc fbc;
         struct intel_opregion opregion;
         struct intel_vbt_data vbt;
@@ -1445,8 +1531,8 @@ typedef struct drm_i915_private {
 
         struct sdvo_device_mapping sdvo_mappings[2];
 
-        struct drm_crtc *plane_to_crtc_mapping[3];
-        struct drm_crtc *pipe_to_crtc_mapping[3];
+        struct drm_crtc *plane_to_crtc_mapping[I915_MAX_PIPES];
+        struct drm_crtc *pipe_to_crtc_mapping[I915_MAX_PIPES];
         wait_queue_head_t pending_flip_queue;
 
 #ifdef CONFIG_DEBUG_FS
@@ -1506,6 +1592,7 @@ typedef struct drm_i915_private {
 
         u32 fdi_rx_config;
 
+        u32 suspend_count;
         struct i915_suspend_saved_registers regfile;
 
         struct {
@@ -1525,8 +1612,6 @@ typedef struct drm_i915_private {
                 struct ilk_wm_values hw;
         } wm;
 
-        struct i915_package_c8 pc8;
-
         struct i915_runtime_pm pm;
 
         /* Old dri1 support infrastructure, beware the dragons ya fools entering
@@ -1627,18 +1712,6 @@ struct drm_i915_gem_object {
          */
         unsigned int fence_dirty:1;
 
-        /** How many users have pinned this object in GTT space. The following
-         * users can each hold at most one reference: pwrite/pread, pin_ioctl
-         * (via user_pin_count), execbuffer (objects are not allowed multiple
-         * times for the same batchbuffer), and the framebuffer code. When
-         * switching/pageflipping, the framebuffer code has at most two buffers
-         * pinned per crtc.
-         *
-         * In the worst case this is 1 + 1 + 1 + 2*2 = 7. That would fit into 3
-         * bits with absolutely no headroom. So use 4 bits. */
-        unsigned int pin_count:4;
-#define DRM_I915_GEM_OBJECT_MAX_PIN_COUNT 0xf
-
         /**
          * Is the object at the current location in the gtt mappable and
          * fenceable? Used to avoid costly recalculations.
@@ -1697,7 +1770,6 @@ struct drm_i915_gem_object {
         /** for phy allocated objects */
         struct drm_i915_gem_phys_object *phys_obj;
 };
-#define to_gem_object(obj) (&((struct drm_i915_gem_object *)(obj))->base)
 
 #define to_intel_bo(x) container_of(x, struct drm_i915_gem_object, base)
 
@@ -1743,6 +1815,7 @@ struct drm_i915_gem_request {
 
 struct drm_i915_file_private {
         struct drm_i915_private *dev_priv;
+        struct drm_file *file;
 
         struct {
                 spinlock_t lock;
@@ -1751,11 +1824,95 @@ struct drm_i915_file_private {
         } mm;
         struct idr context_idr;
 
-        struct i915_ctx_hang_stats hang_stats;
+        struct i915_hw_context *private_default_ctx;
         atomic_t rps_wait_boost;
 };
 
-#define INTEL_INFO(dev) (to_i915(dev)->info)
+/*
+ * A command that requires special handling by the command parser.
+ */
+struct drm_i915_cmd_descriptor {
+        /*
+         * Flags describing how the command parser processes the command.
+         *
+         * CMD_DESC_FIXED: The command has a fixed length if this is set,
+         *                 a length mask if not set
+         * CMD_DESC_SKIP: The command is allowed but does not follow the
+         *                standard length encoding for the opcode range in
+         *                which it falls
+         * CMD_DESC_REJECT: The command is never allowed
+         * CMD_DESC_REGISTER: The command should be checked against the
+         *                    register whitelist for the appropriate ring
+         * CMD_DESC_MASTER: The command is allowed if the submitting process
+         *                  is the DRM master
+         */
+        u32 flags;
+#define CMD_DESC_FIXED    (1<<0)
+#define CMD_DESC_SKIP     (1<<1)
+#define CMD_DESC_REJECT   (1<<2)
+#define CMD_DESC_REGISTER (1<<3)
+#define CMD_DESC_BITMASK  (1<<4)
+#define CMD_DESC_MASTER   (1<<5)
+
+        /*
+         * The command's unique identification bits and the bitmask to get them.
+         * This isn't strictly the opcode field as defined in the spec and may
+         * also include type, subtype, and/or subop fields.
+         */
+        struct {
+                u32 value;
+                u32 mask;
+        } cmd;
+
+        /*
+         * The command's length. The command is either fixed length (i.e. does
+         * not include a length field) or has a length field mask. The flag
+         * CMD_DESC_FIXED indicates a fixed length. Otherwise, the command has
+         * a length mask. All command entries in a command table must include
+         * length information.
+         */
+        union {
+                u32 fixed;
+                u32 mask;
+        } length;
+
+        /*
+         * Describes where to find a register address in the command to check
+         * against the ring's register whitelist. Only valid if flags has the
+         * CMD_DESC_REGISTER bit set.
+         */
+        struct {
+                u32 offset;
+                u32 mask;
+        } reg;
+
+#define MAX_CMD_DESC_BITMASKS 3
+        /*
+         * Describes command checks where a particular dword is masked and
+         * compared against an expected value. If the command does not match
+         * the expected value, the parser rejects it. Only valid if flags has
+         * the CMD_DESC_BITMASK bit set. Only entries where mask is non-zero
+         * are valid.
+         */
+        struct {
+                u32 offset;
+                u32 mask;
+                u32 expected;
+        } bits[MAX_CMD_DESC_BITMASKS];
+};
+
+/*
+ * A table of commands requiring special handling by the command parser.
+ *
+ * Each ring has an array of tables. Each table consists of an array of command
+ * descriptors, which must be sorted with command opcodes in ascending order.
+ */
+struct drm_i915_cmd_table {
+        const struct drm_i915_cmd_descriptor *table;
+        int count;
+};
+
+#define INTEL_INFO(dev) (&to_i915(dev)->info)
 
 #define IS_I830(dev)            ((dev)->pdev->device == 0x3577)
 #define IS_845G(dev)            ((dev)->pdev->device == 0x2562)
@@ -1824,7 +1981,11 @@ struct drm_i915_file_private {
 #define I915_NEED_GFX_HWS(dev)  (INTEL_INFO(dev)->need_gfx_hws)
 
 #define HAS_HW_CONTEXTS(dev)    (INTEL_INFO(dev)->gen >= 6)
-#define HAS_ALIASING_PPGTT(dev) (INTEL_INFO(dev)->gen >=6 && !IS_VALLEYVIEW(dev))
+#define HAS_ALIASING_PPGTT(dev) (INTEL_INFO(dev)->gen >= 6 && !IS_VALLEYVIEW(dev))
+#define HAS_PPGTT(dev)          (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev) \
+                                 && !IS_BROADWELL(dev))
+#define USES_PPGTT(dev)         intel_enable_ppgtt(dev, false)
+#define USES_FULL_PPGTT(dev)    intel_enable_ppgtt(dev, true)
 
 #define HAS_OVERLAY(dev)                (INTEL_INFO(dev)->has_overlay)
 #define OVERLAY_NEEDS_PHYSICAL(dev)     (INTEL_INFO(dev)->overlay_needs_physical)
@@ -1887,32 +2048,40 @@ struct drm_i915_file_private {
 
 extern const struct drm_ioctl_desc i915_ioctls[];
 extern int i915_max_ioctl;
-extern unsigned int i915_fbpercrtc __always_unused;
-extern int i915_panel_ignore_lid __read_mostly;
-extern unsigned int i915_powersave __read_mostly;
-extern int i915_semaphores __read_mostly;
-extern unsigned int i915_lvds_downclock __read_mostly;
-extern int i915_lvds_channel_mode __read_mostly;
-extern int i915_panel_use_ssc __read_mostly;
-extern int i915_vbt_sdvo_panel_type __read_mostly;
-extern int i915_enable_rc6 __read_mostly;
-extern int i915_enable_fbc __read_mostly;
-extern bool i915_enable_hangcheck __read_mostly;
-extern int i915_enable_ppgtt __read_mostly;
-extern int i915_enable_psr __read_mostly;
-extern unsigned int i915_preliminary_hw_support __read_mostly;
-extern int i915_disable_power_well __read_mostly;
-extern int i915_enable_ips __read_mostly;
-extern bool i915_fastboot __read_mostly;
-extern int i915_enable_pc8 __read_mostly;
-extern int i915_pc8_timeout __read_mostly;
-extern bool i915_prefault_disable __read_mostly;
 
 extern int i915_suspend(struct drm_device *dev, pm_message_t state);
 extern int i915_resume(struct drm_device *dev);
 extern int i915_master_create(struct drm_device *dev, struct drm_master *master);
 extern void i915_master_destroy(struct drm_device *dev, struct drm_master *master);
 
+/* i915_params.c */
+struct i915_params {
+        int modeset;
+        int panel_ignore_lid;
+        unsigned int powersave;
+        int semaphores;
+        unsigned int lvds_downclock;
+        int lvds_channel_mode;
+        int panel_use_ssc;
+        int vbt_sdvo_panel_type;
+        int enable_rc6;
+        int enable_fbc;
+        int enable_ppgtt;
+        int enable_psr;
+        unsigned int preliminary_hw_support;
+        int disable_power_well;
+        int enable_ips;
+        int invert_brightness;
+        int enable_cmd_parser;
+        /* leave bools at the end to not create holes */
+        bool enable_hangcheck;
+        bool fastboot;
+        bool prefault_disable;
+        bool reset;
+        bool disable_display;
+};
+extern struct i915_params i915 __read_mostly;
+
                                 /* i915_dma.c */
 void i915_update_dri1_breadcrumb(struct drm_device *dev);
 extern void i915_kernel_lost_context(struct drm_device * dev);
@@ -1943,8 +2112,12 @@ extern void intel_console_resume(struct work_struct *work);
 
 /* i915_irq.c */
 void i915_queue_hangcheck(struct drm_device *dev);
-void i915_handle_error(struct drm_device *dev, bool wedged);
+__printf(3, 4)
+void i915_handle_error(struct drm_device *dev, bool wedged,
+                       const char *fmt, ...);
 
+void gen6_set_pm_mask(struct drm_i915_private *dev_priv, u32 pm_iir,
+                                                        int new_delay);
 extern void intel_irq_init(struct drm_device *dev);
 extern void intel_hpd_init(struct drm_device *dev);
 
@@ -1955,10 +2128,15 @@ extern void intel_uncore_check_errors(struct drm_device *dev);
 extern void intel_uncore_fini(struct drm_device *dev);
 
 void
-i915_enable_pipestat(drm_i915_private_t *dev_priv, enum pipe pipe, u32 mask);
+i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
+                     u32 status_mask);
 
 void
-i915_disable_pipestat(drm_i915_private_t *dev_priv, enum pipe pipe, u32 mask);
+i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
+                      u32 status_mask);
+
+void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv);
+void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv);
 
 /* i915_gem.c */
 int i915_gem_init_ioctl(struct drm_device *dev, void *data,
@@ -2014,22 +2192,27 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
                          const struct drm_i915_gem_object_ops *ops);
 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
                                                   size_t size);
+void i915_init_vm(struct drm_i915_private *dev_priv,
+                  struct i915_address_space *vm);
 void i915_gem_free_object(struct drm_gem_object *obj);
 void i915_gem_vma_destroy(struct i915_vma *vma);
 
+#define PIN_MAPPABLE 0x1
+#define PIN_NONBLOCK 0x2
+#define PIN_GLOBAL 0x4
 int __must_check i915_gem_object_pin(struct drm_i915_gem_object *obj,
                                      struct i915_address_space *vm,
                                      uint32_t alignment,
-                                     bool map_and_fenceable,
-                                     bool nonblocking);
-void i915_gem_object_unpin(struct drm_i915_gem_object *obj);
+                                     unsigned flags);
 int __must_check i915_vma_unbind(struct i915_vma *vma);
-int __must_check i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj);
 int i915_gem_object_put_pages(struct drm_i915_gem_object *obj);
 void i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv);
 void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
 void i915_gem_lastclose(struct drm_device *dev);
 
+int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
+                                    int *needs_clflush);
+
 int __must_check i915_gem_object_get_pages(struct drm_i915_gem_object *obj);
 static inline struct page *i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n)
 {
@@ -2096,8 +2279,10 @@ i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj)
         }
 }
 
+struct drm_i915_gem_request *
+i915_gem_find_active_request(struct intel_ring_buffer *ring);
+
 bool i915_gem_retire_requests(struct drm_device *dev);
-void i915_gem_retire_requests_ring(struct intel_ring_buffer *ring);
 int __must_check i915_gem_check_wedge(struct i915_gpu_error *error,
                                       bool interruptible);
 static inline bool i915_reset_in_progress(struct i915_gpu_error *error)
@@ -2186,6 +2371,13 @@ i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
                                   struct i915_address_space *vm);
 
 struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj);
+static inline bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj) {
+        struct i915_vma *vma;
+        list_for_each_entry(vma, &obj->vma_list, vma_link)
+                if (vma->pin_count > 0)
+                        return true;
+        return false;
+}
 
 /* Some GGTT VM helpers */
 #define obj_to_ggtt(obj) \
@@ -2217,54 +2409,69 @@ i915_gem_obj_ggtt_size(struct drm_i915_gem_object *obj)
 static inline int __must_check
 i915_gem_obj_ggtt_pin(struct drm_i915_gem_object *obj,
                       uint32_t alignment,
-                      bool map_and_fenceable,
-                      bool nonblocking)
+                      unsigned flags)
 {
-        return i915_gem_object_pin(obj, obj_to_ggtt(obj), alignment,
-                                   map_and_fenceable, nonblocking);
+        return i915_gem_object_pin(obj, obj_to_ggtt(obj), alignment, flags | PIN_GLOBAL);
 }
 
+static inline int
+i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
+{
+        return i915_vma_unbind(i915_gem_obj_to_ggtt(obj));
+}
+
+void i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj);
+
 /* i915_gem_context.c */
+#define ctx_to_ppgtt(ctx) container_of((ctx)->vm, struct i915_hw_ppgtt, base)
 int __must_check i915_gem_context_init(struct drm_device *dev);
 void i915_gem_context_fini(struct drm_device *dev);
+void i915_gem_context_reset(struct drm_device *dev);
+int i915_gem_context_open(struct drm_device *dev, struct drm_file *file);
+int i915_gem_context_enable(struct drm_i915_private *dev_priv);
 void i915_gem_context_close(struct drm_device *dev, struct drm_file *file);
 int i915_switch_context(struct intel_ring_buffer *ring,
-                        struct drm_file *file, int to_id);
+                        struct drm_file *file, struct i915_hw_context *to);
+struct i915_hw_context *
+i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id);
 void i915_gem_context_free(struct kref *ctx_ref);
 static inline void i915_gem_context_reference(struct i915_hw_context *ctx)
 {
-        kref_get(&ctx->ref);
+        if (ctx->obj && HAS_HW_CONTEXTS(ctx->obj->base.dev))
+                kref_get(&ctx->ref);
 }
 
 static inline void i915_gem_context_unreference(struct i915_hw_context *ctx)
 {
-        kref_put(&ctx->ref, i915_gem_context_free);
+        if (ctx->obj && HAS_HW_CONTEXTS(ctx->obj->base.dev))
+                kref_put(&ctx->ref, i915_gem_context_free);
+}
+
+static inline bool i915_gem_context_is_default(const struct i915_hw_context *c)
+{
+        return c->id == DEFAULT_CONTEXT_ID;
 }
 
-struct i915_ctx_hang_stats * __must_check
-i915_gem_context_get_hang_stats(struct drm_device *dev,
-                                struct drm_file *file,
-                                u32 id);
 int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
                                   struct drm_file *file);
 int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
                                    struct drm_file *file);
 
-/* i915_gem_gtt.c */
-void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev);
-void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
-                            struct drm_i915_gem_object *obj,
-                            enum i915_cache_level cache_level);
-void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
-                              struct drm_i915_gem_object *obj);
+/* i915_gem_evict.c */
+int __must_check i915_gem_evict_something(struct drm_device *dev,
+                                          struct i915_address_space *vm,
+                                          int min_size,
+                                          unsigned alignment,
+                                          unsigned cache_level,
+                                          unsigned flags);
+int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle);
+int i915_gem_evict_everything(struct drm_device *dev);
 
+/* i915_gem_gtt.c */
 void i915_check_and_clear_faults(struct drm_device *dev);
 void i915_gem_suspend_gtt_mappings(struct drm_device *dev);
 void i915_gem_restore_gtt_mappings(struct drm_device *dev);
 int __must_check i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj);
-void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
-                                enum i915_cache_level cache_level);
-void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj);
 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj);
 void i915_gem_init_global_gtt(struct drm_device *dev);
 void i915_gem_setup_global_gtt(struct drm_device *dev, unsigned long start,
@@ -2275,18 +2482,8 @@ static inline void i915_gem_chipset_flush(struct drm_device *dev)
         if (INTEL_INFO(dev)->gen < 6)
                 intel_gtt_chipset_flush();
 }
-
-
-/* i915_gem_evict.c */
-int __must_check i915_gem_evict_something(struct drm_device *dev,
-                                          struct i915_address_space *vm,
-                                          int min_size,
-                                          unsigned alignment,
-                                          unsigned cache_level,
-                                          bool mappable,
-                                          bool nonblock);
-int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle);
-int i915_gem_evict_everything(struct drm_device *dev);
+int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt);
+bool intel_enable_ppgtt(struct drm_device *dev, bool full);
 
 /* i915_gem_stolen.c */
 int i915_gem_init_stolen(struct drm_device *dev);
@@ -2305,7 +2502,7 @@ void i915_gem_object_release_stolen(struct drm_i915_gem_object *obj);
 /* i915_gem_tiling.c */
 static inline bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 {
-        drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 
         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
                 obj->tiling_mode != I915_TILING_NONE;
@@ -2343,7 +2540,8 @@ static inline void i915_error_state_buf_release(
 {
         kfree(eb->buf);
 }
-void i915_capture_error_state(struct drm_device *dev);
+void i915_capture_error_state(struct drm_device *dev, bool wedge,
+                              const char *error_msg);
 void i915_error_state_get(struct drm_device *dev,
                           struct i915_error_state_file_priv *error_priv);
 void i915_error_state_put(struct i915_error_state_file_priv *error_priv);
@@ -2352,6 +2550,14 @@ void i915_destroy_error_state(struct drm_device *dev);
 void i915_get_extra_instdone(struct drm_device *dev, uint32_t *instdone);
 const char *i915_cache_level_str(int type);
 
+/* i915_cmd_parser.c */
+void i915_cmd_parser_init_ring(struct intel_ring_buffer *ring);
+bool i915_needs_cmd_parser(struct intel_ring_buffer *ring);
+int i915_parse_cmds(struct intel_ring_buffer *ring,
+                    struct drm_i915_gem_object *batch_obj,
+                    u32 batch_start_offset,
+                    bool is_master);
+
 /* i915_suspend.c */
 extern int i915_save_state(struct drm_device *dev);
 extern int i915_restore_state(struct drm_device *dev);
@@ -2425,10 +2631,12 @@ extern void intel_modeset_suspend_hw(struct drm_device *dev);
 extern void intel_modeset_init(struct drm_device *dev);
 extern void intel_modeset_gem_init(struct drm_device *dev);
 extern void intel_modeset_cleanup(struct drm_device *dev);
+extern void intel_connector_unregister(struct intel_connector *);
 extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state);
 extern void intel_modeset_setup_hw_state(struct drm_device *dev,
                                          bool force_restore);
 extern void i915_redisable_vga(struct drm_device *dev);
+extern void i915_redisable_vga_power_on(struct drm_device *dev);
 extern bool intel_fbc_enabled(struct drm_device *dev);
 extern void intel_disable_fbc(struct drm_device *dev);
 extern bool ironlake_set_drps(struct drm_device *dev, u8 val);
@@ -2463,6 +2671,7 @@ extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e,
  */
 void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine);
 void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine);
+void assert_force_wake_inactive(struct drm_i915_private *dev_priv);
 
 int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u8 mbox, u32 *val);
 int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u8 mbox, u32 val);
@@ -2525,9 +2734,26 @@ void vlv_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine);
 #define I915_READ_NOTRACE(reg)          dev_priv->uncore.funcs.mmio_readl(dev_priv, (reg), false)
 #define I915_WRITE_NOTRACE(reg, val)    dev_priv->uncore.funcs.mmio_writel(dev_priv, (reg), (val), false)
 
+/* Be very careful with read/write 64-bit values. On 32-bit machines, they
+ * will be implemented using 2 32-bit writes in an arbitrary order with
+ * an arbitrary delay between them. This can cause the hardware to
+ * act upon the intermediate value, possibly leading to corruption and
+ * machine death. You have been warned.
+ */
 #define I915_WRITE64(reg, val)  dev_priv->uncore.funcs.mmio_writeq(dev_priv, (reg), (val), true)
 #define I915_READ64(reg)        dev_priv->uncore.funcs.mmio_readq(dev_priv, (reg), true)
 
+#define I915_READ64_2x32(lower_reg, upper_reg) ({                       \
+                u32 upper = I915_READ(upper_reg);                       \
+                u32 lower = I915_READ(lower_reg);                       \
+                u32 tmp = I915_READ(upper_reg);                         \
+                if (upper != tmp) {                                     \
+                        upper = tmp;                                    \
+                        lower = I915_READ(lower_reg);                   \
+                        WARN_ON(I915_READ(upper_reg) != upper);         \
+                }                                                       \
+                (u64)upper << 32 | lower; })
+
 #define POSTING_READ(reg)       (void)I915_READ_NOTRACE(reg)
 #define POSTING_READ16(reg)     (void)I915_READ16_NOTRACE(reg)
 
@@ -2566,4 +2792,31 @@ timespec_to_jiffies_timeout(const struct timespec *value)
         return min_t(unsigned long, MAX_JIFFY_OFFSET, j + 1);
 }
 
+/*
+ * If you need to wait X milliseconds between events A and B, but event B
+ * doesn't happen exactly after event A, you record the timestamp (jiffies) of
+ * when event A happened, then just before event B you call this function and
+ * pass the timestamp as the first argument, and X as the second argument.
+ */
+static inline void
+wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms)
+{
+        unsigned long target_jiffies, tmp_jiffies, remaining_jiffies;
+
+        /*
+         * Don't re-read the value of "jiffies" every time since it may change
+         * behind our back and break the math.
+         */
+        tmp_jiffies = jiffies;
+        target_jiffies = timestamp_jiffies +
+                         msecs_to_jiffies_timeout(to_wait_ms);
+
+        if (time_after(target_jiffies, tmp_jiffies)) {
+                remaining_jiffies = target_jiffies - tmp_jiffies;
+                while (remaining_jiffies)
+                        remaining_jiffies =
+                            schedule_timeout_uninterruptible(remaining_jiffies);
+        }
+}
+
 #endif
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 00c836154725..6370a761d137 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -43,12 +43,6 @@ static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *o
 static __must_check int
 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
                                bool readonly);
-static __must_check int
-i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
-                           struct i915_address_space *vm,
-                           unsigned alignment,
-                           bool map_and_fenceable,
-                           bool nonblocking);
 static int i915_gem_phys_pwrite(struct drm_device *dev,
                                 struct drm_i915_gem_object *obj,
                                 struct drm_i915_gem_pwrite *args,
@@ -67,6 +61,7 @@ static unsigned long i915_gem_inactive_scan(struct shrinker *shrinker,
 static unsigned long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
 static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
+static void i915_gem_retire_requests_ring(struct intel_ring_buffer *ring);
 
 static bool cpu_cache_is_coherent(struct drm_device *dev,
                                   enum i915_cache_level level)
@@ -204,7 +199,7 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
         pinned = 0;
         mutex_lock(&dev->struct_mutex);
         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
-                if (obj->pin_count)
+                if (i915_gem_obj_is_pinned(obj))
                         pinned += i915_gem_obj_ggtt_size(obj);
         mutex_unlock(&dev->struct_mutex);
 
@@ -332,6 +327,42 @@ __copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
         return 0;
 }
 
+/*
+ * Pins the specified object's pages and synchronizes the object with
+ * GPU accesses. Sets needs_clflush to non-zero if the caller should
+ * flush the object from the CPU cache.
+ */
+int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
+                                    int *needs_clflush)
+{
+        int ret;
+
+        *needs_clflush = 0;
+
+        if (!obj->base.filp)
+                return -EINVAL;
+
+        if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
+                /* If we're not in the cpu read domain, set ourself into the gtt
+                 * read domain and manually flush cachelines (if required). This
+                 * optimizes for the case when the gpu will dirty the data
+                 * anyway again before the next pread happens. */
+                *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
+                                                        obj->cache_level);
+                ret = i915_gem_object_wait_rendering(obj, true);
+                if (ret)
+                        return ret;
+        }
+
+        ret = i915_gem_object_get_pages(obj);
+        if (ret)
+                return ret;
+
+        i915_gem_object_pin_pages(obj);
+
+        return ret;
+}
+
 /* Per-page copy function for the shmem pread fastpath.
  * Flushes invalid cachelines before reading the target if
  * needs_clflush is set. */
@@ -429,23 +460,10 @@ i915_gem_shmem_pread(struct drm_device *dev,
 
         obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
 
-        if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
-                /* If we're not in the cpu read domain, set ourself into the gtt
-                 * read domain and manually flush cachelines (if required). This
-                 * optimizes for the case when the gpu will dirty the data
-                 * anyway again before the next pread happens. */
-                needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level);
-                ret = i915_gem_object_wait_rendering(obj, true);
-                if (ret)
-                        return ret;
-        }
-
-        ret = i915_gem_object_get_pages(obj);
+        ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
         if (ret)
                 return ret;
 
-        i915_gem_object_pin_pages(obj);
-
         offset = args->offset;
 
         for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
@@ -476,7 +494,7 @@ i915_gem_shmem_pread(struct drm_device *dev,
 
                 mutex_unlock(&dev->struct_mutex);
 
-                if (likely(!i915_prefault_disable) && !prefaulted) {
+                if (likely(!i915.prefault_disable) && !prefaulted) {
                         ret = fault_in_multipages_writeable(user_data, remain);
                         /* Userspace is tricking us, but we've already clobbered
                          * its pages with the prefault and promised to write the
@@ -492,12 +510,10 @@ i915_gem_shmem_pread(struct drm_device *dev,
 
                 mutex_lock(&dev->struct_mutex);
 
-next_page:
-                mark_page_accessed(page);
-
                 if (ret)
                         goto out;
 
+next_page:
                 remain -= page_length;
                 user_data += page_length;
                 offset += page_length;
@@ -599,13 +615,13 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev,
                          struct drm_i915_gem_pwrite *args,
                          struct drm_file *file)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         ssize_t remain;
         loff_t offset, page_base;
         char __user *user_data;
         int page_offset, page_length, ret;
 
-        ret = i915_gem_obj_ggtt_pin(obj, 0, true, true);
+        ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
         if (ret)
                 goto out;
 
@@ -651,7 +667,7 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev,
         }
 
 out_unpin:
-        i915_gem_object_unpin(obj);
+        i915_gem_object_ggtt_unpin(obj);
 out:
         return ret;
 }
@@ -677,9 +693,8 @@ shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
         if (needs_clflush_before)
                 drm_clflush_virt_range(vaddr + shmem_page_offset,
                                        page_length);
-        ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
-                                                user_data,
-                                                page_length);
+        ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
+                                        user_data, page_length);
         if (needs_clflush_after)
                 drm_clflush_virt_range(vaddr + shmem_page_offset,
                                        page_length);
@@ -813,13 +828,10 @@ i915_gem_shmem_pwrite(struct drm_device *dev,
 
                 mutex_lock(&dev->struct_mutex);
 
-next_page:
-                set_page_dirty(page);
-                mark_page_accessed(page);
-
                 if (ret)
                         goto out;
 
+next_page:
                 remain -= page_length;
                 user_data += page_length;
                 offset += page_length;
@@ -868,7 +880,7 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
                        args->size))
                 return -EFAULT;
 
-        if (likely(!i915_prefault_disable)) {
+        if (likely(!i915.prefault_disable)) {
                 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
                                                    args->size);
                 if (ret)
@@ -1014,7 +1026,8 @@ static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
                         struct timespec *timeout,
                         struct drm_i915_file_private *file_priv)
 {
-        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         const bool irq_test_in_progress =
                 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
         struct timespec before, now;
@@ -1022,14 +1035,14 @@ static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
         unsigned long timeout_expire;
         int ret;
 
-        WARN(dev_priv->pc8.irqs_disabled, "IRQs disabled\n");
+        WARN(dev_priv->pm.irqs_disabled, "IRQs disabled\n");
 
         if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
                 return 0;
 
         timeout_expire = timeout ? jiffies + timespec_to_jiffies_timeout(timeout) : 0;
 
-        if (dev_priv->info->gen >= 6 && can_wait_boost(file_priv)) {
+        if (INTEL_INFO(dev)->gen >= 6 && can_wait_boost(file_priv)) {
                 gen6_rps_boost(dev_priv);
                 if (file_priv)
                         mod_delayed_work(dev_priv->wq,
@@ -1184,7 +1197,7 @@ i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
  */
 static __must_check int
 i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
-                                            struct drm_file *file,
+                                            struct drm_i915_file_private *file_priv,
                                             bool readonly)
 {
         struct drm_device *dev = obj->base.dev;
@@ -1211,7 +1224,7 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
 
         reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
         mutex_unlock(&dev->struct_mutex);
-        ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, file->driver_priv);
+        ret = __wait_seqno(ring, seqno, reset_counter, true, NULL, file_priv);
         mutex_lock(&dev->struct_mutex);
         if (ret)
                 return ret;
@@ -1260,7 +1273,9 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
          * We will repeat the flush holding the lock in the normal manner
          * to catch cases where we are gazumped.
          */
-        ret = i915_gem_object_wait_rendering__nonblocking(obj, file, !write_domain);
+        ret = i915_gem_object_wait_rendering__nonblocking(obj,
+                                                          file->driver_priv,
+                                                          !write_domain);
         if (ret)
                 goto unref;
 
@@ -1374,7 +1389,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
         struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
         struct drm_device *dev = obj->base.dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         pgoff_t page_offset;
         unsigned long pfn;
         int ret = 0;
@@ -1392,6 +1407,15 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 
         trace_i915_gem_object_fault(obj, page_offset, true, write);
 
+        /* Try to flush the object off the GPU first without holding the lock.
+         * Upon reacquiring the lock, we will perform our sanity checks and then
+         * repeat the flush holding the lock in the normal manner to catch cases
+         * where we are gazumped.
+         */
+        ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
+        if (ret)
+                goto unlock;
+
         /* Access to snoopable pages through the GTT is incoherent. */
         if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
                 ret = -EINVAL;
@@ -1399,7 +1423,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
         }
 
         /* Now bind it into the GTT if needed */
-        ret = i915_gem_obj_ggtt_pin(obj,  0, true, false);
+        ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
         if (ret)
                 goto unlock;
 
@@ -1420,7 +1444,7 @@ int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
         /* Finally, remap it using the new GTT offset */
         ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
 unpin:
-        i915_gem_object_unpin(obj);
+        i915_gem_object_ggtt_unpin(obj);
 unlock:
         mutex_unlock(&dev->struct_mutex);
 out:
@@ -1453,6 +1477,7 @@ out:
                 ret = VM_FAULT_OOM;
                 break;
         case -ENOSPC:
+        case -EFAULT:
                 ret = VM_FAULT_SIGBUS;
                 break;
         default:
@@ -1501,7 +1526,8 @@ i915_gem_release_mmap(struct drm_i915_gem_object *obj)
         if (!obj->fault_mappable)
                 return;
 
-        drm_vma_node_unmap(&obj->base.vma_node, obj->base.dev->dev_mapping);
+        drm_vma_node_unmap(&obj->base.vma_node,
+                           obj->base.dev->anon_inode->i_mapping);
         obj->fault_mappable = false;
 }
 
@@ -1617,8 +1643,8 @@ i915_gem_mmap_gtt(struct drm_file *file,
         }
 
         if (obj->madv != I915_MADV_WILLNEED) {
-                DRM_ERROR("Attempting to mmap a purgeable buffer\n");
-                ret = -EINVAL;
+                DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
+                ret = -EFAULT;
                 goto out;
         }
 
@@ -1971,8 +1997,8 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
                 return 0;
 
         if (obj->madv != I915_MADV_WILLNEED) {
-                DRM_ERROR("Attempting to obtain a purgeable object\n");
-                return -EINVAL;
+                DRM_DEBUG("Attempting to obtain a purgeable object\n");
+                return -EFAULT;
         }
 
         BUG_ON(obj->pages_pin_count);
@@ -2035,13 +2061,17 @@ static void
 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
 {
         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
-        struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
-        struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
+        struct i915_address_space *vm;
+        struct i915_vma *vma;
 
         BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
         BUG_ON(!obj->active);
 
-        list_move_tail(&vma->mm_list, &ggtt_vm->inactive_list);
+        list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
+                vma = i915_gem_obj_to_vma(obj, vm);
+                if (vma && !list_empty(&vma->mm_list))
+                        list_move_tail(&vma->mm_list, &vm->inactive_list);
+        }
 
         list_del_init(&obj->ring_list);
         obj->ring = NULL;
@@ -2134,10 +2164,9 @@ int __i915_add_request(struct intel_ring_buffer *ring,
                        struct drm_i915_gem_object *obj,
                        u32 *out_seqno)
 {
-        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
         struct drm_i915_gem_request *request;
         u32 request_ring_position, request_start;
-        int was_empty;
         int ret;
 
         request_start = intel_ring_get_tail(ring);
@@ -2188,7 +2217,6 @@ int __i915_add_request(struct intel_ring_buffer *ring,
                 i915_gem_context_reference(request->ctx);
 
         request->emitted_jiffies = jiffies;
-        was_empty = list_empty(&ring->request_list);
         list_add_tail(&request->list, &ring->request_list);
         request->file_priv = NULL;
 
@@ -2209,13 +2237,11 @@ int __i915_add_request(struct intel_ring_buffer *ring,
         if (!dev_priv->ums.mm_suspended) {
                 i915_queue_hangcheck(ring->dev);
 
-                if (was_empty) {
-                        cancel_delayed_work_sync(&dev_priv->mm.idle_work);
-                        queue_delayed_work(dev_priv->wq,
-                                           &dev_priv->mm.retire_work,
-                                           round_jiffies_up_relative(HZ));
-                        intel_mark_busy(dev_priv->dev);
-                }
+                cancel_delayed_work_sync(&dev_priv->mm.idle_work);
+                queue_delayed_work(dev_priv->wq,
+                                   &dev_priv->mm.retire_work,
+                                   round_jiffies_up_relative(HZ));
+                intel_mark_busy(dev_priv->dev);
         }
 
         if (out_seqno)
@@ -2237,125 +2263,46 @@ i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
         spin_unlock(&file_priv->mm.lock);
 }
 
-static bool i915_head_inside_object(u32 acthd, struct drm_i915_gem_object *obj,
-                                    struct i915_address_space *vm)
+static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
+                                   const struct i915_hw_context *ctx)
 {
-        if (acthd >= i915_gem_obj_offset(obj, vm) &&
-            acthd < i915_gem_obj_offset(obj, vm) + obj->base.size)
-                return true;
+        unsigned long elapsed;
 
-        return false;
-}
+        elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
 
-static bool i915_head_inside_request(const u32 acthd_unmasked,
-                                     const u32 request_start,
-                                     const u32 request_end)
-{
-        const u32 acthd = acthd_unmasked & HEAD_ADDR;
+        if (ctx->hang_stats.banned)
+                return true;
 
-        if (request_start < request_end) {
-                if (acthd >= request_start && acthd < request_end)
-                        return true;
-        } else if (request_start > request_end) {
-                if (acthd >= request_start || acthd < request_end)
+        if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
+                if (!i915_gem_context_is_default(ctx)) {
+                        DRM_DEBUG("context hanging too fast, banning!\n");
                         return true;
-        }
-
-        return false;
-}
-
-static struct i915_address_space *
-request_to_vm(struct drm_i915_gem_request *request)
-{
-        struct drm_i915_private *dev_priv = request->ring->dev->dev_private;
-        struct i915_address_space *vm;
-
-        vm = &dev_priv->gtt.base;
-
-        return vm;
-}
-
-static bool i915_request_guilty(struct drm_i915_gem_request *request,
-                                const u32 acthd, bool *inside)
-{
-        /* There is a possibility that unmasked head address
-         * pointing inside the ring, matches the batch_obj address range.
-         * However this is extremely unlikely.
-         */
-        if (request->batch_obj) {
-                if (i915_head_inside_object(acthd, request->batch_obj,
-                                            request_to_vm(request))) {
-                        *inside = true;
+                } else if (dev_priv->gpu_error.stop_rings == 0) {
+                        DRM_ERROR("gpu hanging too fast, banning!\n");
                         return true;
                 }
         }
 
-        if (i915_head_inside_request(acthd, request->head, request->tail)) {
-                *inside = false;
-                return true;
-        }
-
         return false;
 }
 
-static bool i915_context_is_banned(const struct i915_ctx_hang_stats *hs)
+static void i915_set_reset_status(struct drm_i915_private *dev_priv,
+                                  struct i915_hw_context *ctx,
+                                  const bool guilty)
 {
-        const unsigned long elapsed = get_seconds() - hs->guilty_ts;
-
-        if (hs->banned)
-                return true;
-
-        if (elapsed <= DRM_I915_CTX_BAN_PERIOD) {
-                DRM_ERROR("context hanging too fast, declaring banned!\n");
-                return true;
-        }
-
-        return false;
-}
+        struct i915_ctx_hang_stats *hs;
 
-static void i915_set_reset_status(struct intel_ring_buffer *ring,
-                                  struct drm_i915_gem_request *request,
-                                  u32 acthd)
-{
-        struct i915_ctx_hang_stats *hs = NULL;
-        bool inside, guilty;
-        unsigned long offset = 0;
-
-        /* Innocent until proven guilty */
-        guilty = false;
-
-        if (request->batch_obj)
-                offset = i915_gem_obj_offset(request->batch_obj,
-                                             request_to_vm(request));
+        if (WARN_ON(!ctx))
+                return;
 
-        if (ring->hangcheck.action != HANGCHECK_WAIT &&
-            i915_request_guilty(request, acthd, &inside)) {
-                DRM_DEBUG("%s hung %s bo (0x%lx ctx %d) at 0x%x\n",
-                          ring->name,
-                          inside ? "inside" : "flushing",
-                          offset,
-                          request->ctx ? request->ctx->id : 0,
-                          acthd);
+        hs = &ctx->hang_stats;
 
-                guilty = true;
-        }
-
-        /* If contexts are disabled or this is the default context, use
-         * file_priv->reset_state
-         */
-        if (request->ctx && request->ctx->id != DEFAULT_CONTEXT_ID)
-                hs = &request->ctx->hang_stats;
-        else if (request->file_priv)
-                hs = &request->file_priv->hang_stats;
-
-        if (hs) {
-                if (guilty) {
-                        hs->banned = i915_context_is_banned(hs);
-                        hs->batch_active++;
-                        hs->guilty_ts = get_seconds();
-                } else {
-                        hs->batch_pending++;
-                }
+        if (guilty) {
+                hs->banned = i915_context_is_banned(dev_priv, ctx);
+                hs->batch_active++;
+                hs->guilty_ts = get_seconds();
+        } else {
+                hs->batch_pending++;
         }
 }
 
@@ -2370,19 +2317,41 @@ static void i915_gem_free_request(struct drm_i915_gem_request *request)
         kfree(request);
 }
 
-static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
-                                       struct intel_ring_buffer *ring)
+struct drm_i915_gem_request *
+i915_gem_find_active_request(struct intel_ring_buffer *ring)
 {
-        u32 completed_seqno = ring->get_seqno(ring, false);
-        u32 acthd = intel_ring_get_active_head(ring);
         struct drm_i915_gem_request *request;
+        u32 completed_seqno;
+
+        completed_seqno = ring->get_seqno(ring, false);
 
         list_for_each_entry(request, &ring->request_list, list) {
                 if (i915_seqno_passed(completed_seqno, request->seqno))
                         continue;
 
-                i915_set_reset_status(ring, request, acthd);
+                return request;
         }
+
+        return NULL;
+}
+
+static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
+                                       struct intel_ring_buffer *ring)
+{
+        struct drm_i915_gem_request *request;
+        bool ring_hung;
+
+        request = i915_gem_find_active_request(ring);
+
+        if (request == NULL)
+                return;
+
+        ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
+
+        i915_set_reset_status(dev_priv, request->ctx, ring_hung);
+
+        list_for_each_entry_continue(request, &ring->request_list, list)
+                i915_set_reset_status(dev_priv, request->ctx, false);
 }
 
 static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
@@ -2456,13 +2425,15 @@ void i915_gem_reset(struct drm_device *dev)
 
         i915_gem_cleanup_ringbuffer(dev);
 
+        i915_gem_context_reset(dev);
+
         i915_gem_restore_fences(dev);
 }
 
 /**
  * This function clears the request list as sequence numbers are passed.
  */
-void
+static void
 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
 {
         uint32_t seqno;
@@ -2474,6 +2445,24 @@ i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
 
         seqno = ring->get_seqno(ring, true);
 
+        /* Move any buffers on the active list that are no longer referenced
+         * by the ringbuffer to the flushing/inactive lists as appropriate,
+         * before we free the context associated with the requests.
+         */
+        while (!list_empty(&ring->active_list)) {
+                struct drm_i915_gem_object *obj;
+
+                obj = list_first_entry(&ring->active_list,
+                                      struct drm_i915_gem_object,
+                                      ring_list);
+
+                if (!i915_seqno_passed(seqno, obj->last_read_seqno))
+                        break;
+
+                i915_gem_object_move_to_inactive(obj);
+        }
+
+
         while (!list_empty(&ring->request_list)) {
                 struct drm_i915_gem_request *request;
 
@@ -2495,22 +2484,6 @@ i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
                 i915_gem_free_request(request);
         }
 
-        /* Move any buffers on the active list that are no longer referenced
-         * by the ringbuffer to the flushing/inactive lists as appropriate.
-         */
-        while (!list_empty(&ring->active_list)) {
-                struct drm_i915_gem_object *obj;
-
-                obj = list_first_entry(&ring->active_list,
-                                      struct drm_i915_gem_object,
-                                      ring_list);
-
-                if (!i915_seqno_passed(seqno, obj->last_read_seqno))
-                        break;
-
-                i915_gem_object_move_to_inactive(obj);
-        }
-
         if (unlikely(ring->trace_irq_seqno &&
                      i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
                 ring->irq_put(ring);
@@ -2523,7 +2496,7 @@ i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
 bool
 i915_gem_retire_requests(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring;
         bool idle = true;
         int i;
@@ -2615,7 +2588,7 @@ i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
 int
 i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_wait *args = data;
         struct drm_i915_gem_object *obj;
         struct intel_ring_buffer *ring = NULL;
@@ -2750,22 +2723,18 @@ static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
 int i915_vma_unbind(struct i915_vma *vma)
 {
         struct drm_i915_gem_object *obj = vma->obj;
-        drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
         int ret;
 
-        /* For now we only ever use 1 vma per object */
-        WARN_ON(!list_is_singular(&obj->vma_list));
-
         if (list_empty(&vma->vma_link))
                 return 0;
 
         if (!drm_mm_node_allocated(&vma->node)) {
                 i915_gem_vma_destroy(vma);
-
                 return 0;
         }
 
-        if (obj->pin_count)
+        if (vma->pin_count)
                 return -EBUSY;
 
         BUG_ON(obj->pages == NULL);
@@ -2787,15 +2756,11 @@ int i915_vma_unbind(struct i915_vma *vma)
 
         trace_i915_vma_unbind(vma);
 
-        if (obj->has_global_gtt_mapping)
-                i915_gem_gtt_unbind_object(obj);
-        if (obj->has_aliasing_ppgtt_mapping) {
-                i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
-                obj->has_aliasing_ppgtt_mapping = 0;
-        }
+        vma->unbind_vma(vma);
+
         i915_gem_gtt_finish_object(obj);
 
-        list_del(&vma->mm_list);
+        list_del_init(&vma->mm_list);
         /* Avoid an unnecessary call to unbind on rebind. */
         if (i915_is_ggtt(vma->vm))
                 obj->map_and_fenceable = true;
@@ -2817,35 +2782,15 @@ int i915_vma_unbind(struct i915_vma *vma)
         return 0;
 }
 
-/**
- * Unbinds an object from the global GTT aperture.
- */
-int
-i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
-{
-        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
-        struct i915_address_space *ggtt = &dev_priv->gtt.base;
-
-        if (!i915_gem_obj_ggtt_bound(obj))
-                return 0;
-
-        if (obj->pin_count)
-                return -EBUSY;
-
-        BUG_ON(obj->pages == NULL);
-
-        return i915_vma_unbind(i915_gem_obj_to_vma(obj, ggtt));
-}
-
 int i915_gpu_idle(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring;
         int ret, i;
 
         /* Flush everything onto the inactive list. */
         for_each_ring(ring, dev_priv, i) {
-                ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
+                ret = i915_switch_context(ring, NULL, ring->default_context);
                 if (ret)
                         return ret;
 
@@ -2860,7 +2805,7 @@ int i915_gpu_idle(struct drm_device *dev)
 static void i965_write_fence_reg(struct drm_device *dev, int reg,
                                  struct drm_i915_gem_object *obj)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int fence_reg;
         int fence_pitch_shift;
 
@@ -2912,7 +2857,7 @@ static void i965_write_fence_reg(struct drm_device *dev, int reg,
 static void i915_write_fence_reg(struct drm_device *dev, int reg,
                                  struct drm_i915_gem_object *obj)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 val;
 
         if (obj) {
@@ -2956,7 +2901,7 @@ static void i915_write_fence_reg(struct drm_device *dev, int reg,
 static void i830_write_fence_reg(struct drm_device *dev, int reg,
                                 struct drm_i915_gem_object *obj)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t val;
 
         if (obj) {
@@ -3259,18 +3204,17 @@ static void i915_gem_verify_gtt(struct drm_device *dev)
 /**
  * Finds free space in the GTT aperture and binds the object there.
  */
-static int
+static struct i915_vma *
 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
                            struct i915_address_space *vm,
                            unsigned alignment,
-                           bool map_and_fenceable,
-                           bool nonblocking)
+                           unsigned flags)
 {
         struct drm_device *dev = obj->base.dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 size, fence_size, fence_alignment, unfenced_alignment;
         size_t gtt_max =
-                map_and_fenceable ? dev_priv->gtt.mappable_end : vm->total;
+                flags & PIN_MAPPABLE ? dev_priv->gtt.mappable_end : vm->total;
         struct i915_vma *vma;
         int ret;
 
@@ -3282,57 +3226,49 @@ i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
                                                      obj->tiling_mode, true);
         unfenced_alignment =
                 i915_gem_get_gtt_alignment(dev,
-                                                    obj->base.size,
-                                                    obj->tiling_mode, false);
+                                           obj->base.size,
+                                           obj->tiling_mode, false);
 
         if (alignment == 0)
-                alignment = map_and_fenceable ? fence_alignment :
+                alignment = flags & PIN_MAPPABLE ? fence_alignment :
                                                 unfenced_alignment;
-        if (map_and_fenceable && alignment & (fence_alignment - 1)) {
-                DRM_ERROR("Invalid object alignment requested %u\n", alignment);
-                return -EINVAL;
+        if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
+                DRM_DEBUG("Invalid object alignment requested %u\n", alignment);
+                return ERR_PTR(-EINVAL);
         }
 
-        size = map_and_fenceable ? fence_size : obj->base.size;
+        size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
 
         /* If the object is bigger than the entire aperture, reject it early
          * before evicting everything in a vain attempt to find space.
          */
         if (obj->base.size > gtt_max) {
-                DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
+                DRM_DEBUG("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%zu\n",
                           obj->base.size,
-                          map_and_fenceable ? "mappable" : "total",
+                          flags & PIN_MAPPABLE ? "mappable" : "total",
                           gtt_max);
-                return -E2BIG;
+                return ERR_PTR(-E2BIG);
         }
 
         ret = i915_gem_object_get_pages(obj);
         if (ret)
-                return ret;
+                return ERR_PTR(ret);
 
         i915_gem_object_pin_pages(obj);
 
-        BUG_ON(!i915_is_ggtt(vm));
-
         vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
-        if (IS_ERR(vma)) {
-                ret = PTR_ERR(vma);
+        if (IS_ERR(vma))
                 goto err_unpin;
-        }
-
-        /* For now we only ever use 1 vma per object */
-        WARN_ON(!list_is_singular(&obj->vma_list));
 
 search_free:
         ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
                                                   size, alignment,
                                                   obj->cache_level, 0, gtt_max,
-                                                  DRM_MM_SEARCH_DEFAULT);
+                                                  DRM_MM_SEARCH_DEFAULT,
+                                                  DRM_MM_CREATE_DEFAULT);
         if (ret) {
                 ret = i915_gem_evict_something(dev, vm, size, alignment,
-                                               obj->cache_level,
-                                               map_and_fenceable,
-                                               nonblocking);
+                                               obj->cache_level, flags);
                 if (ret == 0)
                         goto search_free;
 
@@ -3363,19 +3299,23 @@ search_free:
                 obj->map_and_fenceable = mappable && fenceable;
         }
 
-        WARN_ON(map_and_fenceable && !obj->map_and_fenceable);
+        WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
+
+        trace_i915_vma_bind(vma, flags);
+        vma->bind_vma(vma, obj->cache_level,
+                      flags & (PIN_MAPPABLE | PIN_GLOBAL) ? GLOBAL_BIND : 0);
 
-        trace_i915_vma_bind(vma, map_and_fenceable);
         i915_gem_verify_gtt(dev);
-        return 0;
+        return vma;
 
 err_remove_node:
         drm_mm_remove_node(&vma->node);
 err_free_vma:
         i915_gem_vma_destroy(vma);
+        vma = ERR_PTR(ret);
 err_unpin:
         i915_gem_object_unpin_pages(obj);
-        return ret;
+        return vma;
 }
 
 bool
@@ -3470,7 +3410,7 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj,
 int
 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 {
-        drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
         uint32_t old_write_domain, old_read_domains;
         int ret;
 
@@ -3528,25 +3468,22 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
                                     enum i915_cache_level cache_level)
 {
         struct drm_device *dev = obj->base.dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        struct i915_vma *vma;
+        struct i915_vma *vma, *next;
         int ret;
 
         if (obj->cache_level == cache_level)
                 return 0;
 
-        if (obj->pin_count) {
+        if (i915_gem_obj_is_pinned(obj)) {
                 DRM_DEBUG("can not change the cache level of pinned objects\n");
                 return -EBUSY;
         }
 
-        list_for_each_entry(vma, &obj->vma_list, vma_link) {
+        list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
                 if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
                         ret = i915_vma_unbind(vma);
                         if (ret)
                                 return ret;
-
-                        break;
                 }
         }
 
@@ -3567,11 +3504,10 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
                                 return ret;
                 }
 
-                if (obj->has_global_gtt_mapping)
-                        i915_gem_gtt_bind_object(obj, cache_level);
-                if (obj->has_aliasing_ppgtt_mapping)
-                        i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
-                                               obj, cache_level);
+                list_for_each_entry(vma, &obj->vma_list, vma_link)
+                        if (drm_mm_node_allocated(&vma->node))
+                                vma->bind_vma(vma, cache_level,
+                                              obj->has_global_gtt_mapping ? GLOBAL_BIND : 0);
         }
 
         list_for_each_entry(vma, &obj->vma_list, vma_link)
@@ -3695,7 +3631,7 @@ static bool is_pin_display(struct drm_i915_gem_object *obj)
          * subtracting the potential reference by the user, any pin_count
          * remains, it must be due to another use by the display engine.
          */
-        return obj->pin_count - !!obj->user_pin_count;
+        return i915_gem_obj_to_ggtt(obj)->pin_count - !!obj->user_pin_count;
 }
 
 /*
@@ -3740,7 +3676,7 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
          * (e.g. libkms for the bootup splash), we have to ensure that we
          * always use map_and_fenceable for all scanout buffers.
          */
-        ret = i915_gem_obj_ggtt_pin(obj, alignment, true, false);
+        ret = i915_gem_obj_ggtt_pin(obj, alignment, PIN_MAPPABLE);
         if (ret)
                 goto err_unpin_display;
 
@@ -3769,7 +3705,7 @@ err_unpin_display:
 void
 i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
 {
-        i915_gem_object_unpin(obj);
+        i915_gem_object_ggtt_unpin(obj);
         obj->pin_display = is_pin_display(obj);
 }
 
@@ -3896,65 +3832,63 @@ int
 i915_gem_object_pin(struct drm_i915_gem_object *obj,
                     struct i915_address_space *vm,
                     uint32_t alignment,
-                    bool map_and_fenceable,
-                    bool nonblocking)
+                    unsigned flags)
 {
         struct i915_vma *vma;
         int ret;
 
-        if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
-                return -EBUSY;
-
-        WARN_ON(map_and_fenceable && !i915_is_ggtt(vm));
+        if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
+                return -EINVAL;
 
         vma = i915_gem_obj_to_vma(obj, vm);
-
         if (vma) {
+                if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
+                        return -EBUSY;
+
                 if ((alignment &&
                      vma->node.start & (alignment - 1)) ||
-                    (map_and_fenceable && !obj->map_and_fenceable)) {
-                        WARN(obj->pin_count,
+                    (flags & PIN_MAPPABLE && !obj->map_and_fenceable)) {
+                        WARN(vma->pin_count,
                              "bo is already pinned with incorrect alignment:"
                              " offset=%lx, req.alignment=%x, req.map_and_fenceable=%d,"
                              " obj->map_and_fenceable=%d\n",
                              i915_gem_obj_offset(obj, vm), alignment,
-                             map_and_fenceable,
+                             flags & PIN_MAPPABLE,
                              obj->map_and_fenceable);
                         ret = i915_vma_unbind(vma);
                         if (ret)
                                 return ret;
+
+                        vma = NULL;
                 }
         }
 
-        if (!i915_gem_obj_bound(obj, vm)) {
-                struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
-
-                ret = i915_gem_object_bind_to_vm(obj, vm, alignment,
-                                                 map_and_fenceable,
-                                                 nonblocking);
-                if (ret)
-                        return ret;
-
-                if (!dev_priv->mm.aliasing_ppgtt)
-                        i915_gem_gtt_bind_object(obj, obj->cache_level);
+        if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
+                vma = i915_gem_object_bind_to_vm(obj, vm, alignment, flags);
+                if (IS_ERR(vma))
+                        return PTR_ERR(vma);
         }
 
-        if (!obj->has_global_gtt_mapping && map_and_fenceable)
-                i915_gem_gtt_bind_object(obj, obj->cache_level);
+        if (flags & PIN_GLOBAL && !obj->has_global_gtt_mapping)
+                vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
 
-        obj->pin_count++;
-        obj->pin_mappable |= map_and_fenceable;
+        vma->pin_count++;
+        if (flags & PIN_MAPPABLE)
+                obj->pin_mappable |= true;
 
         return 0;
 }
 
 void
-i915_gem_object_unpin(struct drm_i915_gem_object *obj)
+i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj)
 {
-        BUG_ON(obj->pin_count == 0);
-        BUG_ON(!i915_gem_obj_bound_any(obj));
+        struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
 
-        if (--obj->pin_count == 0)
+        BUG_ON(!vma);
+        BUG_ON(vma->pin_count == 0);
+        BUG_ON(!i915_gem_obj_ggtt_bound(obj));
+
+        if (--vma->pin_count == 0)
                 obj->pin_mappable = false;
 }
 
@@ -3966,6 +3900,9 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
         struct drm_i915_gem_object *obj;
         int ret;
 
+        if (INTEL_INFO(dev)->gen >= 6)
+                return -ENODEV;
+
         ret = i915_mutex_lock_interruptible(dev);
         if (ret)
                 return ret;
@@ -3977,13 +3914,13 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
         }
 
         if (obj->madv != I915_MADV_WILLNEED) {
-                DRM_ERROR("Attempting to pin a purgeable buffer\n");
-                ret = -EINVAL;
+                DRM_DEBUG("Attempting to pin a purgeable buffer\n");
+                ret = -EFAULT;
                 goto out;
         }
 
         if (obj->pin_filp != NULL && obj->pin_filp != file) {
-                DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
+                DRM_DEBUG("Already pinned in i915_gem_pin_ioctl(): %d\n",
                           args->handle);
                 ret = -EINVAL;
                 goto out;
@@ -3995,7 +3932,7 @@ i915_gem_pin_ioctl(struct drm_device *dev, void *data,
         }
 
         if (obj->user_pin_count == 0) {
-                ret = i915_gem_obj_ggtt_pin(obj, args->alignment, true, false);
+                ret = i915_gem_obj_ggtt_pin(obj, args->alignment, PIN_MAPPABLE);
                 if (ret)
                         goto out;
         }
@@ -4030,7 +3967,7 @@ i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
         }
 
         if (obj->pin_filp != file) {
-                DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
+                DRM_DEBUG("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
                           args->handle);
                 ret = -EINVAL;
                 goto out;
@@ -4038,7 +3975,7 @@ i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
         obj->user_pin_count--;
         if (obj->user_pin_count == 0) {
                 obj->pin_filp = NULL;
-                i915_gem_object_unpin(obj);
+                i915_gem_object_ggtt_unpin(obj);
         }
 
 out:
@@ -4118,7 +4055,7 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
                 goto unlock;
         }
 
-        if (obj->pin_count) {
+        if (i915_gem_obj_is_pinned(obj)) {
                 ret = -EINVAL;
                 goto out;
         }
@@ -4219,7 +4156,7 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
 {
         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
         struct drm_device *dev = obj->base.dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct i915_vma *vma, *next;
 
         intel_runtime_pm_get(dev_priv);
@@ -4229,12 +4166,11 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
         if (obj->phys_obj)
                 i915_gem_detach_phys_object(dev, obj);
 
-        obj->pin_count = 0;
-        /* NB: 0 or 1 elements */
-        WARN_ON(!list_empty(&obj->vma_list) &&
-                !list_is_singular(&obj->vma_list));
         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
-                int ret = i915_vma_unbind(vma);
+                int ret;
+
+                vma->pin_count = 0;
+                ret = i915_vma_unbind(vma);
                 if (WARN_ON(ret == -ERESTARTSYS)) {
                         bool was_interruptible;
 
@@ -4283,41 +4219,6 @@ struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
         return NULL;
 }
 
-static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
-                                              struct i915_address_space *vm)
-{
-        struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
-        if (vma == NULL)
-                return ERR_PTR(-ENOMEM);
-
-        INIT_LIST_HEAD(&vma->vma_link);
-        INIT_LIST_HEAD(&vma->mm_list);
-        INIT_LIST_HEAD(&vma->exec_list);
-        vma->vm = vm;
-        vma->obj = obj;
-
-        /* Keep GGTT vmas first to make debug easier */
-        if (i915_is_ggtt(vm))
-                list_add(&vma->vma_link, &obj->vma_list);
-        else
-                list_add_tail(&vma->vma_link, &obj->vma_list);
-
-        return vma;
-}
-
-struct i915_vma *
-i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
-                                  struct i915_address_space *vm)
-{
-        struct i915_vma *vma;
-
-        vma = i915_gem_obj_to_vma(obj, vm);
-        if (!vma)
-                vma = __i915_gem_vma_create(obj, vm);
-
-        return vma;
-}
-
 void i915_gem_vma_destroy(struct i915_vma *vma)
 {
         WARN_ON(vma->node.allocated);
@@ -4334,7 +4235,7 @@ void i915_gem_vma_destroy(struct i915_vma *vma)
 int
 i915_gem_suspend(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret = 0;
 
         mutex_lock(&dev->struct_mutex);
@@ -4376,7 +4277,7 @@ err:
 int i915_gem_l3_remap(struct intel_ring_buffer *ring, int slice)
 {
         struct drm_device *dev = ring->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 reg_base = GEN7_L3LOG_BASE + (slice * 0x200);
         u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
         int i, ret;
@@ -4406,7 +4307,7 @@ int i915_gem_l3_remap(struct intel_ring_buffer *ring, int slice)
 
 void i915_gem_init_swizzling(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         if (INTEL_INFO(dev)->gen < 5 ||
             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
@@ -4494,7 +4395,7 @@ cleanup_render_ring:
 int
 i915_gem_init_hw(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret, i;
 
         if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
@@ -4508,9 +4409,15 @@ i915_gem_init_hw(struct drm_device *dev)
                            LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
 
         if (HAS_PCH_NOP(dev)) {
-                u32 temp = I915_READ(GEN7_MSG_CTL);
-                temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
-                I915_WRITE(GEN7_MSG_CTL, temp);
+                if (IS_IVYBRIDGE(dev)) {
+                        u32 temp = I915_READ(GEN7_MSG_CTL);
+                        temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
+                        I915_WRITE(GEN7_MSG_CTL, temp);
+                } else if (INTEL_INFO(dev)->gen >= 7) {
+                        u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
+                        temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
+                        I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
+                }
         }
 
         i915_gem_init_swizzling(dev);
@@ -4523,25 +4430,23 @@ i915_gem_init_hw(struct drm_device *dev)
                 i915_gem_l3_remap(&dev_priv->ring[RCS], i);
 
         /*
-         * XXX: There was some w/a described somewhere suggesting loading
-         * contexts before PPGTT.
+         * XXX: Contexts should only be initialized once. Doing a switch to the
+         * default context switch however is something we'd like to do after
+         * reset or thaw (the latter may not actually be necessary for HW, but
+         * goes with our code better). Context switching requires rings (for
+         * the do_switch), but before enabling PPGTT. So don't move this.
          */
-        ret = i915_gem_context_init(dev);
+        ret = i915_gem_context_enable(dev_priv);
         if (ret) {
-                i915_gem_cleanup_ringbuffer(dev);
-                DRM_ERROR("Context initialization failed %d\n", ret);
-                return ret;
-        }
-
-        if (dev_priv->mm.aliasing_ppgtt) {
-                ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
-                if (ret) {
-                        i915_gem_cleanup_aliasing_ppgtt(dev);
-                        DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
-                }
+                DRM_ERROR("Context enable failed %d\n", ret);
+                goto err_out;
         }
 
         return 0;
+
+err_out:
+        i915_gem_cleanup_ringbuffer(dev);
+        return ret;
 }
 
 int i915_gem_init(struct drm_device *dev)
@@ -4560,10 +4465,18 @@ int i915_gem_init(struct drm_device *dev)
 
         i915_gem_init_global_gtt(dev);
 
+        ret = i915_gem_context_init(dev);
+        if (ret) {
+                mutex_unlock(&dev->struct_mutex);
+                return ret;
+        }
+
         ret = i915_gem_init_hw(dev);
         mutex_unlock(&dev->struct_mutex);
         if (ret) {
-                i915_gem_cleanup_aliasing_ppgtt(dev);
+                WARN_ON(dev_priv->mm.aliasing_ppgtt);
+                i915_gem_context_fini(dev);
+                drm_mm_takedown(&dev_priv->gtt.base.mm);
                 return ret;
         }
 
@@ -4576,7 +4489,7 @@ int i915_gem_init(struct drm_device *dev)
 void
 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring;
         int i;
 
@@ -4658,20 +4571,22 @@ init_ring_lists(struct intel_ring_buffer *ring)
         INIT_LIST_HEAD(&ring->request_list);
 }
 
-static void i915_init_vm(struct drm_i915_private *dev_priv,
-                         struct i915_address_space *vm)
+void i915_init_vm(struct drm_i915_private *dev_priv,
+                  struct i915_address_space *vm)
 {
+        if (!i915_is_ggtt(vm))
+                drm_mm_init(&vm->mm, vm->start, vm->total);
         vm->dev = dev_priv->dev;
         INIT_LIST_HEAD(&vm->active_list);
         INIT_LIST_HEAD(&vm->inactive_list);
         INIT_LIST_HEAD(&vm->global_link);
-        list_add(&vm->global_link, &dev_priv->vm_list);
+        list_add_tail(&vm->global_link, &dev_priv->vm_list);
 }
 
 void
 i915_gem_load(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int i;
 
         dev_priv->slab =
@@ -4738,7 +4653,7 @@ i915_gem_load(struct drm_device *dev)
 static int i915_gem_init_phys_object(struct drm_device *dev,
                                      int id, int size, int align)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_phys_object *phys_obj;
         int ret;
 
@@ -4770,7 +4685,7 @@ kfree_obj:
 
 static void i915_gem_free_phys_object(struct drm_device *dev, int id)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_phys_object *phys_obj;
 
         if (!dev_priv->mm.phys_objs[id - 1])
@@ -4837,7 +4752,7 @@ i915_gem_attach_phys_object(struct drm_device *dev,
                             int align)
 {
         struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret = 0;
         int page_count;
         int i;
@@ -4950,6 +4865,7 @@ i915_gem_file_idle_work_handler(struct work_struct *work)
 int i915_gem_open(struct drm_device *dev, struct drm_file *file)
 {
         struct drm_i915_file_private *file_priv;
+        int ret;
 
         DRM_DEBUG_DRIVER("\n");
 
@@ -4959,15 +4875,18 @@ int i915_gem_open(struct drm_device *dev, struct drm_file *file)
 
         file->driver_priv = file_priv;
         file_priv->dev_priv = dev->dev_private;
+        file_priv->file = file;
 
         spin_lock_init(&file_priv->mm.lock);
         INIT_LIST_HEAD(&file_priv->mm.request_list);
         INIT_DELAYED_WORK(&file_priv->mm.idle_work,
                           i915_gem_file_idle_work_handler);
 
-        idr_init(&file_priv->context_idr);
+        ret = i915_gem_context_open(dev, file);
+        if (ret)
+                kfree(file_priv);
 
-        return 0;
+        return ret;
 }
 
 static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
@@ -5014,7 +4933,7 @@ i915_gem_inactive_count(struct shrinker *shrinker, struct shrink_control *sc)
                 if (obj->active)
                         continue;
 
-                if (obj->pin_count == 0 && obj->pages_pin_count == 0)
+                if (!i915_gem_obj_is_pinned(obj) && obj->pages_pin_count == 0)
                         count += obj->base.size >> PAGE_SHIFT;
         }
 
@@ -5031,7 +4950,8 @@ unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
         struct i915_vma *vma;
 
-        if (vm == &dev_priv->mm.aliasing_ppgtt->base)
+        if (!dev_priv->mm.aliasing_ppgtt ||
+            vm == &dev_priv->mm.aliasing_ppgtt->base)
                 vm = &dev_priv->gtt.base;
 
         BUG_ON(list_empty(&o->vma_list));
@@ -5072,7 +4992,8 @@ unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
         struct i915_vma *vma;
 
-        if (vm == &dev_priv->mm.aliasing_ppgtt->base)
+        if (!dev_priv->mm.aliasing_ppgtt ||
+            vm == &dev_priv->mm.aliasing_ppgtt->base)
                 vm = &dev_priv->gtt.base;
 
         BUG_ON(list_empty(&o->vma_list));
@@ -5127,7 +5048,7 @@ struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj)
                 return NULL;
 
         vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link);
-        if (WARN_ON(vma->vm != obj_to_ggtt(obj)))
+        if (vma->vm != obj_to_ggtt(obj))
                 return NULL;
 
         return vma;
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index e08acaba5402..6043062ffce7 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -93,11 +93,63 @@
  * I've seen in a spec to date, and that was a workaround for a non-shipping
  * part. It should be safe to decrease this, but it's more future proof as is.
  */
-#define CONTEXT_ALIGN (64<<10)
+#define GEN6_CONTEXT_ALIGN (64<<10)
+#define GEN7_CONTEXT_ALIGN 4096
 
-static struct i915_hw_context *
-i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id);
-static int do_switch(struct i915_hw_context *to);
+static int do_switch(struct intel_ring_buffer *ring,
+                     struct i915_hw_context *to);
+
+static void do_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt)
+{
+        struct drm_device *dev = ppgtt->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct i915_address_space *vm = &ppgtt->base;
+
+        if (ppgtt == dev_priv->mm.aliasing_ppgtt ||
+            (list_empty(&vm->active_list) && list_empty(&vm->inactive_list))) {
+                ppgtt->base.cleanup(&ppgtt->base);
+                return;
+        }
+
+        /*
+         * Make sure vmas are unbound before we take down the drm_mm
+         *
+         * FIXME: Proper refcounting should take care of this, this shouldn't be
+         * needed at all.
+         */
+        if (!list_empty(&vm->active_list)) {
+                struct i915_vma *vma;
+
+                list_for_each_entry(vma, &vm->active_list, mm_list)
+                        if (WARN_ON(list_empty(&vma->vma_link) ||
+                                    list_is_singular(&vma->vma_link)))
+                                break;
+
+                i915_gem_evict_vm(&ppgtt->base, true);
+        } else {
+                i915_gem_retire_requests(dev);
+                i915_gem_evict_vm(&ppgtt->base, false);
+        }
+
+        ppgtt->base.cleanup(&ppgtt->base);
+}
+
+static void ppgtt_release(struct kref *kref)
+{
+        struct i915_hw_ppgtt *ppgtt =
+                container_of(kref, struct i915_hw_ppgtt, ref);
+
+        do_ppgtt_cleanup(ppgtt);
+        kfree(ppgtt);
+}
+
+static size_t get_context_alignment(struct drm_device *dev)
+{
+        if (IS_GEN6(dev))
+                return GEN6_CONTEXT_ALIGN;
+
+        return GEN7_CONTEXT_ALIGN;
+}
 
 static int get_context_size(struct drm_device *dev)
 {
@@ -131,14 +183,44 @@ void i915_gem_context_free(struct kref *ctx_ref)
 {
         struct i915_hw_context *ctx = container_of(ctx_ref,
                                                    typeof(*ctx), ref);
+        struct i915_hw_ppgtt *ppgtt = NULL;
 
-        list_del(&ctx->link);
+        /* We refcount even the aliasing PPGTT to keep the code symmetric */
+        if (USES_PPGTT(ctx->obj->base.dev))
+                ppgtt = ctx_to_ppgtt(ctx);
+
+        /* XXX: Free up the object before tearing down the address space, in
+         * case we're bound in the PPGTT */
         drm_gem_object_unreference(&ctx->obj->base);
+
+        if (ppgtt)
+                kref_put(&ppgtt->ref, ppgtt_release);
+        list_del(&ctx->link);
         kfree(ctx);
 }
 
+static struct i915_hw_ppgtt *
+create_vm_for_ctx(struct drm_device *dev, struct i915_hw_context *ctx)
+{
+        struct i915_hw_ppgtt *ppgtt;
+        int ret;
+
+        ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
+        if (!ppgtt)
+                return ERR_PTR(-ENOMEM);
+
+        ret = i915_gem_init_ppgtt(dev, ppgtt);
+        if (ret) {
+                kfree(ppgtt);
+                return ERR_PTR(ret);
+        }
+
+        ppgtt->ctx = ctx;
+        return ppgtt;
+}
+
 static struct i915_hw_context *
-create_hw_context(struct drm_device *dev,
+__create_hw_context(struct drm_device *dev,
                   struct drm_i915_file_private *file_priv)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -166,18 +248,13 @@ create_hw_context(struct drm_device *dev,
                         goto err_out;
         }
 
-        /* The ring associated with the context object is handled by the normal
-         * object tracking code. We give an initial ring value simple to pass an
-         * assertion in the context switch code.
-         */
-        ctx->ring = &dev_priv->ring[RCS];
         list_add_tail(&ctx->link, &dev_priv->context_list);
 
         /* Default context will never have a file_priv */
         if (file_priv == NULL)
                 return ctx;
 
-        ret = idr_alloc(&file_priv->context_idr, ctx, DEFAULT_CONTEXT_ID + 1, 0,
+        ret = idr_alloc(&file_priv->context_idr, ctx, DEFAULT_CONTEXT_ID, 0,
                         GFP_KERNEL);
         if (ret < 0)
                 goto err_out;
@@ -196,67 +273,136 @@ err_out:
         return ERR_PTR(ret);
 }
 
-static inline bool is_default_context(struct i915_hw_context *ctx)
-{
-        return (ctx == ctx->ring->default_context);
-}
-
 /**
  * The default context needs to exist per ring that uses contexts. It stores the
  * context state of the GPU for applications that don't utilize HW contexts, as
  * well as an idle case.
  */
-static int create_default_context(struct drm_i915_private *dev_priv)
+static struct i915_hw_context *
+i915_gem_create_context(struct drm_device *dev,
+                        struct drm_i915_file_private *file_priv,
+                        bool create_vm)
 {
+        const bool is_global_default_ctx = file_priv == NULL;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct i915_hw_context *ctx;
-        int ret;
+        int ret = 0;
 
-        BUG_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
+        BUG_ON(!mutex_is_locked(&dev->struct_mutex));
 
-        ctx = create_hw_context(dev_priv->dev, NULL);
+        ctx = __create_hw_context(dev, file_priv);
         if (IS_ERR(ctx))
-                return PTR_ERR(ctx);
-
-        /* We may need to do things with the shrinker which require us to
-         * immediately switch back to the default context. This can cause a
-         * problem as pinning the default context also requires GTT space which
-         * may not be available. To avoid this we always pin the
-         * default context.
-         */
-        ret = i915_gem_obj_ggtt_pin(ctx->obj, CONTEXT_ALIGN, false, false);
-        if (ret) {
-                DRM_DEBUG_DRIVER("Couldn't pin %d\n", ret);
-                goto err_destroy;
-        }
+                return ctx;
 
-        ret = do_switch(ctx);
-        if (ret) {
-                DRM_DEBUG_DRIVER("Switch failed %d\n", ret);
-                goto err_unpin;
+        if (is_global_default_ctx) {
+                /* We may need to do things with the shrinker which
+                 * require us to immediately switch back to the default
+                 * context. This can cause a problem as pinning the
+                 * default context also requires GTT space which may not
+                 * be available. To avoid this we always pin the default
+                 * context.
+                 */
+                ret = i915_gem_obj_ggtt_pin(ctx->obj,
+                                            get_context_alignment(dev), 0);
+                if (ret) {
+                        DRM_DEBUG_DRIVER("Couldn't pin %d\n", ret);
+                        goto err_destroy;
+                }
         }
 
-        dev_priv->ring[RCS].default_context = ctx;
+        if (create_vm) {
+                struct i915_hw_ppgtt *ppgtt = create_vm_for_ctx(dev, ctx);
+
+                if (IS_ERR_OR_NULL(ppgtt)) {
+                        DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n",
+                                         PTR_ERR(ppgtt));
+                        ret = PTR_ERR(ppgtt);
+                        goto err_unpin;
+                } else
+                        ctx->vm = &ppgtt->base;
+
+                /* This case is reserved for the global default context and
+                 * should only happen once. */
+                if (is_global_default_ctx) {
+                        if (WARN_ON(dev_priv->mm.aliasing_ppgtt)) {
+                                ret = -EEXIST;
+                                goto err_unpin;
+                        }
+
+                        dev_priv->mm.aliasing_ppgtt = ppgtt;
+                }
+        } else if (USES_PPGTT(dev)) {
+                /* For platforms which only have aliasing PPGTT, we fake the
+                 * address space and refcounting. */
+                ctx->vm = &dev_priv->mm.aliasing_ppgtt->base;
+                kref_get(&dev_priv->mm.aliasing_ppgtt->ref);
+        } else
+                ctx->vm = &dev_priv->gtt.base;
 
-        DRM_DEBUG_DRIVER("Default HW context loaded\n");
-        return 0;
+        return ctx;
 
 err_unpin:
-        i915_gem_object_unpin(ctx->obj);
+        if (is_global_default_ctx)
+                i915_gem_object_ggtt_unpin(ctx->obj);
 err_destroy:
         i915_gem_context_unreference(ctx);
-        return ret;
+        return ERR_PTR(ret);
+}
+
+void i915_gem_context_reset(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring;
+        int i;
+
+        if (!HAS_HW_CONTEXTS(dev))
+                return;
+
+        /* Prevent the hardware from restoring the last context (which hung) on
+         * the next switch */
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                struct i915_hw_context *dctx;
+                if (!(INTEL_INFO(dev)->ring_mask & (1<<i)))
+                        continue;
+
+                /* Do a fake switch to the default context */
+                ring = &dev_priv->ring[i];
+                dctx = ring->default_context;
+                if (WARN_ON(!dctx))
+                        continue;
+
+                if (!ring->last_context)
+                        continue;
+
+                if (ring->last_context == dctx)
+                        continue;
+
+                if (i == RCS) {
+                        WARN_ON(i915_gem_obj_ggtt_pin(dctx->obj,
+                                                      get_context_alignment(dev), 0));
+                        /* Fake a finish/inactive */
+                        dctx->obj->base.write_domain = 0;
+                        dctx->obj->active = 0;
+                }
+
+                i915_gem_context_unreference(ring->last_context);
+                i915_gem_context_reference(dctx);
+                ring->last_context = dctx;
+        }
 }
 
 int i915_gem_context_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int ret;
+        struct intel_ring_buffer *ring;
+        int i;
 
         if (!HAS_HW_CONTEXTS(dev))
                 return 0;
 
-        /* If called from reset, or thaw... we've been here already */
-        if (dev_priv->ring[RCS].default_context)
+        /* Init should only be called once per module load. Eventually the
+         * restriction on the context_disabled check can be loosened. */
+        if (WARN_ON(dev_priv->ring[RCS].default_context))
                 return 0;
 
         dev_priv->hw_context_size = round_up(get_context_size(dev), 4096);
@@ -266,11 +412,23 @@ int i915_gem_context_init(struct drm_device *dev)
                 return -E2BIG;
         }
 
-        ret = create_default_context(dev_priv);
-        if (ret) {
-                DRM_DEBUG_DRIVER("Disabling HW Contexts; create failed %d\n",
-                                 ret);
-                return ret;
+        dev_priv->ring[RCS].default_context =
+                i915_gem_create_context(dev, NULL, USES_PPGTT(dev));
+
+        if (IS_ERR_OR_NULL(dev_priv->ring[RCS].default_context)) {
+                DRM_DEBUG_DRIVER("Disabling HW Contexts; create failed %ld\n",
+                                 PTR_ERR(dev_priv->ring[RCS].default_context));
+                return PTR_ERR(dev_priv->ring[RCS].default_context);
+        }
+
+        for (i = RCS + 1; i < I915_NUM_RINGS; i++) {
+                if (!(INTEL_INFO(dev)->ring_mask & (1<<i)))
+                        continue;
+
+                ring = &dev_priv->ring[i];
+
+                /* NB: RCS will hold a ref for all rings */
+                ring->default_context = dev_priv->ring[RCS].default_context;
         }
 
         DRM_DEBUG_DRIVER("HW context support initialized\n");
@@ -281,6 +439,7 @@ void i915_gem_context_fini(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct i915_hw_context *dctx = dev_priv->ring[RCS].default_context;
+        int i;
 
         if (!HAS_HW_CONTEXTS(dev))
                 return;
@@ -300,59 +459,129 @@ void i915_gem_context_fini(struct drm_device *dev)
         if (dev_priv->ring[RCS].last_context == dctx) {
                 /* Fake switch to NULL context */
                 WARN_ON(dctx->obj->active);
-                i915_gem_object_unpin(dctx->obj);
+                i915_gem_object_ggtt_unpin(dctx->obj);
                 i915_gem_context_unreference(dctx);
+                dev_priv->ring[RCS].last_context = NULL;
         }
 
-        i915_gem_object_unpin(dctx->obj);
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                struct intel_ring_buffer *ring = &dev_priv->ring[i];
+                if (!(INTEL_INFO(dev)->ring_mask & (1<<i)))
+                        continue;
+
+                if (ring->last_context)
+                        i915_gem_context_unreference(ring->last_context);
+
+                ring->default_context = NULL;
+                ring->last_context = NULL;
+        }
+
+        i915_gem_object_ggtt_unpin(dctx->obj);
         i915_gem_context_unreference(dctx);
-        dev_priv->ring[RCS].default_context = NULL;
-        dev_priv->ring[RCS].last_context = NULL;
+        dev_priv->mm.aliasing_ppgtt = NULL;
+}
+
+int i915_gem_context_enable(struct drm_i915_private *dev_priv)
+{
+        struct intel_ring_buffer *ring;
+        int ret, i;
+
+        if (!HAS_HW_CONTEXTS(dev_priv->dev))
+                return 0;
+
+        /* This is the only place the aliasing PPGTT gets enabled, which means
+         * it has to happen before we bail on reset */
+        if (dev_priv->mm.aliasing_ppgtt) {
+                struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
+                ppgtt->enable(ppgtt);
+        }
+
+        /* FIXME: We should make this work, even in reset */
+        if (i915_reset_in_progress(&dev_priv->gpu_error))
+                return 0;
+
+        BUG_ON(!dev_priv->ring[RCS].default_context);
+
+        for_each_ring(ring, dev_priv, i) {
+                ret = do_switch(ring, ring->default_context);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
 }
 
 static int context_idr_cleanup(int id, void *p, void *data)
 {
         struct i915_hw_context *ctx = p;
 
-        BUG_ON(id == DEFAULT_CONTEXT_ID);
+        /* Ignore the default context because close will handle it */
+        if (i915_gem_context_is_default(ctx))
+                return 0;
 
         i915_gem_context_unreference(ctx);
         return 0;
 }
 
-struct i915_ctx_hang_stats *
-i915_gem_context_get_hang_stats(struct drm_device *dev,
-                                struct drm_file *file,
-                                u32 id)
+int i915_gem_context_open(struct drm_device *dev, struct drm_file *file)
 {
         struct drm_i915_file_private *file_priv = file->driver_priv;
-        struct i915_hw_context *ctx;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
-        if (id == DEFAULT_CONTEXT_ID)
-                return &file_priv->hang_stats;
+        if (!HAS_HW_CONTEXTS(dev)) {
+                /* Cheat for hang stats */
+                file_priv->private_default_ctx =
+                        kzalloc(sizeof(struct i915_hw_context), GFP_KERNEL);
 
-        if (!HAS_HW_CONTEXTS(dev))
-                return ERR_PTR(-ENOENT);
+                if (file_priv->private_default_ctx == NULL)
+                        return -ENOMEM;
 
-        ctx = i915_gem_context_get(file->driver_priv, id);
-        if (ctx == NULL)
-                return ERR_PTR(-ENOENT);
+                file_priv->private_default_ctx->vm = &dev_priv->gtt.base;
+                return 0;
+        }
+
+        idr_init(&file_priv->context_idr);
 
-        return &ctx->hang_stats;
+        mutex_lock(&dev->struct_mutex);
+        file_priv->private_default_ctx =
+                i915_gem_create_context(dev, file_priv, USES_FULL_PPGTT(dev));
+        mutex_unlock(&dev->struct_mutex);
+
+        if (IS_ERR(file_priv->private_default_ctx)) {
+                idr_destroy(&file_priv->context_idr);
+                return PTR_ERR(file_priv->private_default_ctx);
+        }
+
+        return 0;
 }
 
 void i915_gem_context_close(struct drm_device *dev, struct drm_file *file)
 {
         struct drm_i915_file_private *file_priv = file->driver_priv;
 
+        if (!HAS_HW_CONTEXTS(dev)) {
+                kfree(file_priv->private_default_ctx);
+                return;
+        }
+
         idr_for_each(&file_priv->context_idr, context_idr_cleanup, NULL);
+        i915_gem_context_unreference(file_priv->private_default_ctx);
         idr_destroy(&file_priv->context_idr);
 }
 
-static struct i915_hw_context *
+struct i915_hw_context *
 i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id)
 {
-        return (struct i915_hw_context *)idr_find(&file_priv->context_idr, id);
+        struct i915_hw_context *ctx;
+
+        if (!HAS_HW_CONTEXTS(file_priv->dev_priv->dev))
+                return file_priv->private_default_ctx;
+
+        ctx = (struct i915_hw_context *)idr_find(&file_priv->context_idr, id);
+        if (!ctx)
+                return ERR_PTR(-ENOENT);
+
+        return ctx;
 }
 
 static inline int
@@ -390,7 +619,10 @@ mi_set_context(struct intel_ring_buffer *ring,
                         MI_SAVE_EXT_STATE_EN |
                         MI_RESTORE_EXT_STATE_EN |
                         hw_flags);
-        /* w/a: MI_SET_CONTEXT must always be followed by MI_NOOP */
+        /*
+         * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP
+         * WaMiSetContext_Hang:snb,ivb,vlv
+         */
         intel_ring_emit(ring, MI_NOOP);
 
         if (IS_GEN7(ring->dev))
@@ -403,21 +635,30 @@ mi_set_context(struct intel_ring_buffer *ring,
         return ret;
 }
 
-static int do_switch(struct i915_hw_context *to)
+static int do_switch(struct intel_ring_buffer *ring,
+                     struct i915_hw_context *to)
 {
-        struct intel_ring_buffer *ring = to->ring;
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
         struct i915_hw_context *from = ring->last_context;
+        struct i915_hw_ppgtt *ppgtt = ctx_to_ppgtt(to);
         u32 hw_flags = 0;
         int ret, i;
 
-        BUG_ON(from != NULL && from->obj != NULL && from->obj->pin_count == 0);
+        if (from != NULL && ring == &dev_priv->ring[RCS]) {
+                BUG_ON(from->obj == NULL);
+                BUG_ON(!i915_gem_obj_is_pinned(from->obj));
+        }
 
-        if (from == to && !to->remap_slice)
+        if (from == to && from->last_ring == ring && !to->remap_slice)
                 return 0;
 
-        ret = i915_gem_obj_ggtt_pin(to->obj, CONTEXT_ALIGN, false, false);
-        if (ret)
-                return ret;
+        /* Trying to pin first makes error handling easier. */
+        if (ring == &dev_priv->ring[RCS]) {
+                ret = i915_gem_obj_ggtt_pin(to->obj,
+                                            get_context_alignment(ring->dev), 0);
+                if (ret)
+                        return ret;
+        }
 
         /*
          * Pin can switch back to the default context if we end up calling into
@@ -426,6 +667,18 @@ static int do_switch(struct i915_hw_context *to)
          */
         from = ring->last_context;
 
+        if (USES_FULL_PPGTT(ring->dev)) {
+                ret = ppgtt->switch_mm(ppgtt, ring, false);
+                if (ret)
+                        goto unpin_out;
+        }
+
+        if (ring != &dev_priv->ring[RCS]) {
+                if (from)
+                        i915_gem_context_unreference(from);
+                goto done;
+        }
+
         /*
          * Clear this page out of any CPU caches for coherent swap-in/out. Note
          * that thanks to write = false in this call and us not setting any gpu
@@ -435,22 +688,21 @@ static int do_switch(struct i915_hw_context *to)
          * XXX: We need a real interface to do this instead of trickery.
          */
         ret = i915_gem_object_set_to_gtt_domain(to->obj, false);
-        if (ret) {
-                i915_gem_object_unpin(to->obj);
-                return ret;
-        }
+        if (ret)
+                goto unpin_out;
 
-        if (!to->obj->has_global_gtt_mapping)
-                i915_gem_gtt_bind_object(to->obj, to->obj->cache_level);
+        if (!to->obj->has_global_gtt_mapping) {
+                struct i915_vma *vma = i915_gem_obj_to_vma(to->obj,
+                                                           &dev_priv->gtt.base);
+                vma->bind_vma(vma, to->obj->cache_level, GLOBAL_BIND);
+        }
 
-        if (!to->is_initialized || is_default_context(to))
+        if (!to->is_initialized || i915_gem_context_is_default(to))
                 hw_flags |= MI_RESTORE_INHIBIT;
 
         ret = mi_set_context(ring, to, hw_flags);
-        if (ret) {
-                i915_gem_object_unpin(to->obj);
-                return ret;
-        }
+        if (ret)
+                goto unpin_out;
 
         for (i = 0; i < MAX_L3_SLICES; i++) {
                 if (!(to->remap_slice & (1<<i)))
@@ -484,22 +736,30 @@ static int do_switch(struct i915_hw_context *to)
                 BUG_ON(from->obj->ring != ring);
 
                 /* obj is kept alive until the next request by its active ref */
-                i915_gem_object_unpin(from->obj);
+                i915_gem_object_ggtt_unpin(from->obj);
                 i915_gem_context_unreference(from);
         }
 
+        to->is_initialized = true;
+
+done:
         i915_gem_context_reference(to);
         ring->last_context = to;
-        to->is_initialized = true;
+        to->last_ring = ring;
 
         return 0;
+
+unpin_out:
+        if (ring->id == RCS)
+                i915_gem_object_ggtt_unpin(to->obj);
+        return ret;
 }
 
 /**
  * i915_switch_context() - perform a GPU context switch.
  * @ring: ring for which we'll execute the context switch
  * @file_priv: file_priv associated with the context, may be NULL
- * @id: context id number
+ * @to: the context to switch to
  *
  * The context life cycle is simple. The context refcount is incremented and
  * decremented by 1 and create and destroy. If the context is in use by the GPU,
@@ -508,31 +768,21 @@ static int do_switch(struct i915_hw_context *to)
  */
 int i915_switch_context(struct intel_ring_buffer *ring,
                         struct drm_file *file,
-                        int to_id)
+                        struct i915_hw_context *to)
 {
         struct drm_i915_private *dev_priv = ring->dev->dev_private;
-        struct i915_hw_context *to;
-
-        if (!HAS_HW_CONTEXTS(ring->dev))
-                return 0;
 
         WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
 
-        if (ring != &dev_priv->ring[RCS])
-                return 0;
-
-        if (to_id == DEFAULT_CONTEXT_ID) {
-                to = ring->default_context;
-        } else {
-                if (file == NULL)
-                        return -EINVAL;
+        BUG_ON(file && to == NULL);
 
-                to = i915_gem_context_get(file->driver_priv, to_id);
-                if (to == NULL)
-                        return -ENOENT;
+        /* We have the fake context */
+        if (!HAS_HW_CONTEXTS(ring->dev)) {
+                ring->last_context = to;
+                return 0;
         }
 
-        return do_switch(to);
+        return do_switch(ring, to);
 }
 
 int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
@@ -543,9 +793,6 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
         struct i915_hw_context *ctx;
         int ret;
 
-        if (!(dev->driver->driver_features & DRIVER_GEM))
-                return -ENODEV;
-
         if (!HAS_HW_CONTEXTS(dev))
                 return -ENODEV;
 
@@ -553,7 +800,7 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
         if (ret)
                 return ret;
 
-        ctx = create_hw_context(dev, file_priv);
+        ctx = i915_gem_create_context(dev, file_priv, USES_FULL_PPGTT(dev));
         mutex_unlock(&dev->struct_mutex);
         if (IS_ERR(ctx))
                 return PTR_ERR(ctx);
@@ -572,17 +819,17 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
         struct i915_hw_context *ctx;
         int ret;
 
-        if (!(dev->driver->driver_features & DRIVER_GEM))
-                return -ENODEV;
+        if (args->ctx_id == DEFAULT_CONTEXT_ID)
+                return -ENOENT;
 
         ret = i915_mutex_lock_interruptible(dev);
         if (ret)
                 return ret;
 
         ctx = i915_gem_context_get(file_priv, args->ctx_id);
-        if (!ctx) {
+        if (IS_ERR(ctx)) {
                 mutex_unlock(&dev->struct_mutex);
-                return -ENOENT;
+                return PTR_ERR(ctx);
         }
 
         idr_remove(&ctx->file_priv->context_idr, ctx->id);
diff --git a/drivers/gpu/drm/i915/i915_gem_debug.c b/drivers/gpu/drm/i915/i915_gem_debug.c
index 775d506b3208..f462d1b51d97 100644
--- a/drivers/gpu/drm/i915/i915_gem_debug.c
+++ b/drivers/gpu/drm/i915/i915_gem_debug.c
@@ -34,7 +34,7 @@ int
 i915_verify_lists(struct drm_device *dev)
 {
         static int warned;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_object *obj;
         int err = 0;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 2ca280f9ee53..75fca63dc8c1 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -36,7 +36,7 @@
 static bool
 mark_free(struct i915_vma *vma, struct list_head *unwind)
 {
-        if (vma->obj->pin_count)
+        if (vma->pin_count)
                 return false;
 
         if (WARN_ON(!list_empty(&vma->exec_list)))
@@ -46,18 +46,37 @@ mark_free(struct i915_vma *vma, struct list_head *unwind)
         return drm_mm_scan_add_block(&vma->node);
 }
 
+/**
+ * i915_gem_evict_something - Evict vmas to make room for binding a new one
+ * @dev: drm_device
+ * @vm: address space to evict from
+ * @size: size of the desired free space
+ * @alignment: alignment constraint of the desired free space
+ * @cache_level: cache_level for the desired space
+ * @mappable: whether the free space must be mappable
+ * @nonblocking: whether evicting active objects is allowed or not
+ *
+ * This function will try to evict vmas until a free space satisfying the
+ * requirements is found. Callers must check first whether any such hole exists
+ * already before calling this function.
+ *
+ * This function is used by the object/vma binding code.
+ *
+ * To clarify: This is for freeing up virtual address space, not for freeing
+ * memory in e.g. the shrinker.
+ */
 int
 i915_gem_evict_something(struct drm_device *dev, struct i915_address_space *vm,
                          int min_size, unsigned alignment, unsigned cache_level,
-                         bool mappable, bool nonblocking)
+                         unsigned flags)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct list_head eviction_list, unwind_list;
         struct i915_vma *vma;
         int ret = 0;
         int pass = 0;
 
-        trace_i915_gem_evict(dev, min_size, alignment, mappable);
+        trace_i915_gem_evict(dev, min_size, alignment, flags);
 
         /*
          * The goal is to evict objects and amalgamate space in LRU order.
@@ -83,7 +102,7 @@ i915_gem_evict_something(struct drm_device *dev, struct i915_address_space *vm,
          */
 
         INIT_LIST_HEAD(&unwind_list);
-        if (mappable) {
+        if (flags & PIN_MAPPABLE) {
                 BUG_ON(!i915_is_ggtt(vm));
                 drm_mm_init_scan_with_range(&vm->mm, min_size,
                                             alignment, cache_level, 0,
@@ -98,7 +117,7 @@ search_again:
                         goto found;
         }
 
-        if (nonblocking)
+        if (flags & PIN_NONBLOCK)
                 goto none;
 
         /* Now merge in the soon-to-be-expired objects... */
@@ -122,7 +141,7 @@ none:
         /* Can we unpin some objects such as idle hw contents,
          * or pending flips?
          */
-        if (nonblocking)
+        if (flags & PIN_NONBLOCK)
                 return -ENOSPC;
 
         /* Only idle the GPU and repeat the search once */
@@ -177,19 +196,19 @@ found:
 }
 
 /**
- * i915_gem_evict_vm - Try to free up VM space
+ * i915_gem_evict_vm - Evict all idle vmas from a vm
  *
- * @vm: Address space to evict from
+ * @vm: Address space to cleanse
  * @do_idle: Boolean directing whether to idle first.
  *
- * VM eviction is about freeing up virtual address space. If one wants fine
- * grained eviction, they should see evict something for more details. In terms
- * of freeing up actual system memory, this function may not accomplish the
- * desired result. An object may be shared in multiple address space, and this
- * function will not assert those objects be freed.
+ * This function evicts all idles vmas from a vm. If all unpinned vmas should be
+ * evicted the @do_idle needs to be set to true.
  *
- * Using do_idle will result in a more complete eviction because it retires, and
- * inactivates current BOs.
+ * This is used by the execbuf code as a last-ditch effort to defragment the
+ * address space.
+ *
+ * To clarify: This is for freeing up virtual address space, not for freeing
+ * memory in e.g. the shrinker.
  */
 int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)
 {
@@ -207,16 +226,24 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)
         }
 
         list_for_each_entry_safe(vma, next, &vm->inactive_list, mm_list)
-                if (vma->obj->pin_count == 0)
+                if (vma->pin_count == 0)
                         WARN_ON(i915_vma_unbind(vma));
 
         return 0;
 }
 
+/**
+ * i915_gem_evict_everything - Try to evict all objects
+ * @dev: Device to evict objects for
+ *
+ * This functions tries to evict all gem objects from all address spaces. Used
+ * by the shrinker as a last-ditch effort and for suspend, before releasing the
+ * backing storage of all unbound objects.
+ */
 int
 i915_gem_evict_everything(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct i915_address_space *vm;
         bool lists_empty = true;
         int ret;
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index d269ecf46e26..7447160155a3 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -91,6 +91,7 @@ eb_lookup_vmas(struct eb_vmas *eb,
                struct i915_address_space *vm,
                struct drm_file *file)
 {
+        struct drm_i915_private *dev_priv = vm->dev->dev_private;
         struct drm_i915_gem_object *obj;
         struct list_head objects;
         int i, ret;
@@ -125,6 +126,20 @@ eb_lookup_vmas(struct eb_vmas *eb,
         i = 0;
         while (!list_empty(&objects)) {
                 struct i915_vma *vma;
+                struct i915_address_space *bind_vm = vm;
+
+                if (exec[i].flags & EXEC_OBJECT_NEEDS_GTT &&
+                    USES_FULL_PPGTT(vm->dev)) {
+                        ret = -EINVAL;
+                        goto err;
+                }
+
+                /* If we have secure dispatch, or the userspace assures us that
+                 * they know what they're doing, use the GGTT VM.
+                 */
+                if (((args->flags & I915_EXEC_SECURE) &&
+                    (i == (args->buffer_count - 1))))
+                        bind_vm = &dev_priv->gtt.base;
 
                 obj = list_first_entry(&objects,
                                        struct drm_i915_gem_object,
@@ -138,7 +153,7 @@ eb_lookup_vmas(struct eb_vmas *eb,
                  * from the (obj, vm) we don't run the risk of creating
                  * duplicated vmas for the same vm.
                  */
-                vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
+                vma = i915_gem_obj_lookup_or_create_vma(obj, bind_vm);
                 if (IS_ERR(vma)) {
                         DRM_DEBUG("Failed to lookup VMA\n");
                         ret = PTR_ERR(vma);
@@ -217,7 +232,7 @@ i915_gem_execbuffer_unreserve_vma(struct i915_vma *vma)
                 i915_gem_object_unpin_fence(obj);
 
         if (entry->flags & __EXEC_OBJECT_HAS_PIN)
-                i915_gem_object_unpin(obj);
+                vma->pin_count--;
 
         entry->flags &= ~(__EXEC_OBJECT_HAS_FENCE | __EXEC_OBJECT_HAS_PIN);
 }
@@ -327,8 +342,7 @@ relocate_entry_gtt(struct drm_i915_gem_object *obj,
 static int
 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
                                    struct eb_vmas *eb,
-                                   struct drm_i915_gem_relocation_entry *reloc,
-                                   struct i915_address_space *vm)
+                                   struct drm_i915_gem_relocation_entry *reloc)
 {
         struct drm_device *dev = obj->base.dev;
         struct drm_gem_object *target_obj;
@@ -352,8 +366,10 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
         if (unlikely(IS_GEN6(dev) &&
             reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
             !target_i915_obj->has_global_gtt_mapping)) {
-                i915_gem_gtt_bind_object(target_i915_obj,
-                                         target_i915_obj->cache_level);
+                struct i915_vma *vma =
+                        list_first_entry(&target_i915_obj->vma_list,
+                                         typeof(*vma), vma_link);
+                vma->bind_vma(vma, target_i915_obj->cache_level, GLOBAL_BIND);
         }
 
         /* Validate that the target is in a valid r/w GPU domain */
@@ -451,8 +467,7 @@ i915_gem_execbuffer_relocate_vma(struct i915_vma *vma,
                 do {
                         u64 offset = r->presumed_offset;
 
-                        ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r,
-                                                                 vma->vm);
+                        ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, r);
                         if (ret)
                                 return ret;
 
@@ -481,8 +496,7 @@ i915_gem_execbuffer_relocate_vma_slow(struct i915_vma *vma,
         int i, ret;
 
         for (i = 0; i < entry->relocation_count; i++) {
-                ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i],
-                                                         vma->vm);
+                ret = i915_gem_execbuffer_relocate_entry(vma->obj, eb, &relocs[i]);
                 if (ret)
                         return ret;
         }
@@ -527,21 +541,26 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
                                 struct intel_ring_buffer *ring,
                                 bool *need_reloc)
 {
-        struct drm_i915_private *dev_priv = ring->dev->dev_private;
+        struct drm_i915_gem_object *obj = vma->obj;
         struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
         bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
-        bool need_fence, need_mappable;
-        struct drm_i915_gem_object *obj = vma->obj;
+        bool need_fence;
+        unsigned flags;
         int ret;
 
+        flags = 0;
+
         need_fence =
                 has_fenced_gpu_access &&
                 entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
                 obj->tiling_mode != I915_TILING_NONE;
-        need_mappable = need_fence || need_reloc_mappable(vma);
+        if (need_fence || need_reloc_mappable(vma))
+                flags |= PIN_MAPPABLE;
 
-        ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, need_mappable,
-                                  false);
+        if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
+                flags |= PIN_GLOBAL;
+
+        ret = i915_gem_object_pin(obj, vma->vm, entry->alignment, flags);
         if (ret)
                 return ret;
 
@@ -560,14 +579,6 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
                 }
         }
 
-        /* Ensure ppgtt mapping exists if needed */
-        if (dev_priv->mm.aliasing_ppgtt && !obj->has_aliasing_ppgtt_mapping) {
-                i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
-                                       obj, obj->cache_level);
-
-                obj->has_aliasing_ppgtt_mapping = 1;
-        }
-
         if (entry->offset != vma->node.start) {
                 entry->offset = vma->node.start;
                 *need_reloc = true;
@@ -578,10 +589,6 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
                 obj->base.pending_write_domain = I915_GEM_DOMAIN_RENDER;
         }
 
-        if (entry->flags & EXEC_OBJECT_NEEDS_GTT &&
-            !obj->has_global_gtt_mapping)
-                i915_gem_gtt_bind_object(obj, obj->cache_level);
-
         return 0;
 }
 
@@ -891,7 +898,7 @@ validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
                 if (!access_ok(VERIFY_WRITE, ptr, length))
                         return -EFAULT;
 
-                if (likely(!i915_prefault_disable)) {
+                if (likely(!i915.prefault_disable)) {
                         if (fault_in_multipages_readable(ptr, length))
                                 return -EFAULT;
                 }
@@ -900,22 +907,27 @@ validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
         return 0;
 }
 
-static int
+static struct i915_hw_context *
 i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
-                          const u32 ctx_id)
+                          struct intel_ring_buffer *ring, const u32 ctx_id)
 {
+        struct i915_hw_context *ctx = NULL;
         struct i915_ctx_hang_stats *hs;
 
-        hs = i915_gem_context_get_hang_stats(dev, file, ctx_id);
-        if (IS_ERR(hs))
-                return PTR_ERR(hs);
+        if (ring->id != RCS && ctx_id != DEFAULT_CONTEXT_ID)
+                return ERR_PTR(-EINVAL);
+
+        ctx = i915_gem_context_get(file->driver_priv, ctx_id);
+        if (IS_ERR(ctx))
+                return ctx;
 
+        hs = &ctx->hang_stats;
         if (hs->banned) {
                 DRM_DEBUG("Context %u tried to submit while banned\n", ctx_id);
-                return -EIO;
+                return ERR_PTR(-EIO);
         }
 
-        return 0;
+        return ctx;
 }
 
 static void
@@ -939,7 +951,9 @@ i915_gem_execbuffer_move_to_active(struct list_head *vmas,
                 if (obj->base.write_domain) {
                         obj->dirty = 1;
                         obj->last_write_seqno = intel_ring_get_seqno(ring);
-                        if (obj->pin_count) /* check for potential scanout */
+                        /* check for potential scanout */
+                        if (i915_gem_obj_ggtt_bound(obj) &&
+                            i915_gem_obj_to_ggtt(obj)->pin_count)
                                 intel_mark_fb_busy(obj, ring);
                 }
 
@@ -964,7 +978,7 @@ static int
 i915_reset_gen7_sol_offsets(struct drm_device *dev,
                             struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret, i;
 
         if (!IS_GEN7(dev) || ring != &dev_priv->ring[RCS])
@@ -989,16 +1003,17 @@ static int
 i915_gem_do_execbuffer(struct drm_device *dev, void *data,
                        struct drm_file *file,
                        struct drm_i915_gem_execbuffer2 *args,
-                       struct drm_i915_gem_exec_object2 *exec,
-                       struct i915_address_space *vm)
+                       struct drm_i915_gem_exec_object2 *exec)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct eb_vmas *eb;
         struct drm_i915_gem_object *batch_obj;
         struct drm_clip_rect *cliprects = NULL;
         struct intel_ring_buffer *ring;
+        struct i915_hw_context *ctx;
+        struct i915_address_space *vm;
         const u32 ctx_id = i915_execbuffer2_get_context_id(*args);
-        u32 exec_start, exec_len;
+        u32 exec_start = args->batch_start_offset, exec_len;
         u32 mask, flags;
         int ret, mode, i;
         bool need_relocs;
@@ -1020,41 +1035,17 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
         if (args->flags & I915_EXEC_IS_PINNED)
                 flags |= I915_DISPATCH_PINNED;
 
-        switch (args->flags & I915_EXEC_RING_MASK) {
-        case I915_EXEC_DEFAULT:
-        case I915_EXEC_RENDER:
-                ring = &dev_priv->ring[RCS];
-                break;
-        case I915_EXEC_BSD:
-                ring = &dev_priv->ring[VCS];
-                if (ctx_id != DEFAULT_CONTEXT_ID) {
-                        DRM_DEBUG("Ring %s doesn't support contexts\n",
-                                  ring->name);
-                        return -EPERM;
-                }
-                break;
-        case I915_EXEC_BLT:
-                ring = &dev_priv->ring[BCS];
-                if (ctx_id != DEFAULT_CONTEXT_ID) {
-                        DRM_DEBUG("Ring %s doesn't support contexts\n",
-                                  ring->name);
-                        return -EPERM;
-                }
-                break;
-        case I915_EXEC_VEBOX:
-                ring = &dev_priv->ring[VECS];
-                if (ctx_id != DEFAULT_CONTEXT_ID) {
-                        DRM_DEBUG("Ring %s doesn't support contexts\n",
-                                  ring->name);
-                        return -EPERM;
-                }
-                break;
-
-        default:
+        if ((args->flags & I915_EXEC_RING_MASK) > I915_NUM_RINGS) {
                 DRM_DEBUG("execbuf with unknown ring: %d\n",
                           (int)(args->flags & I915_EXEC_RING_MASK));
                 return -EINVAL;
         }
+
+        if ((args->flags & I915_EXEC_RING_MASK) == I915_EXEC_DEFAULT)
+                ring = &dev_priv->ring[RCS];
+        else
+                ring = &dev_priv->ring[(args->flags & I915_EXEC_RING_MASK) - 1];
+
         if (!intel_ring_initialized(ring)) {
                 DRM_DEBUG("execbuf with invalid ring: %d\n",
                           (int)(args->flags & I915_EXEC_RING_MASK));
@@ -1136,11 +1127,18 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
                 goto pre_mutex_err;
         }
 
-        ret = i915_gem_validate_context(dev, file, ctx_id);
-        if (ret) {
+        ctx = i915_gem_validate_context(dev, file, ring, ctx_id);
+        if (IS_ERR(ctx)) {
                 mutex_unlock(&dev->struct_mutex);
+                ret = PTR_ERR(ctx);
                 goto pre_mutex_err;
-        }
+        } 
+
+        i915_gem_context_reference(ctx);
+
+        vm = ctx->vm;
+        if (!USES_FULL_PPGTT(dev))
+                vm = &dev_priv->gtt.base;
 
         eb = eb_create(args);
         if (eb == NULL) {
@@ -1184,17 +1182,46 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
         }
         batch_obj->base.pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
 
+        if (i915_needs_cmd_parser(ring)) {
+                ret = i915_parse_cmds(ring,
+                                      batch_obj,
+                                      args->batch_start_offset,
+                                      file->is_master);
+                if (ret)
+                        goto err;
+
+                /*
+                 * XXX: Actually do this when enabling batch copy...
+                 *
+                 * Set the DISPATCH_SECURE bit to remove the NON_SECURE bit
+                 * from MI_BATCH_BUFFER_START commands issued in the
+                 * dispatch_execbuffer implementations. We specifically don't
+                 * want that set when the command parser is enabled.
+                 */
+        }
+
         /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
          * batch" bit. Hence we need to pin secure batches into the global gtt.
          * hsw should have this fixed, but bdw mucks it up again. */
-        if (flags & I915_DISPATCH_SECURE && !batch_obj->has_global_gtt_mapping)
-                i915_gem_gtt_bind_object(batch_obj, batch_obj->cache_level);
+        if (flags & I915_DISPATCH_SECURE &&
+            !batch_obj->has_global_gtt_mapping) {
+                /* When we have multiple VMs, we'll need to make sure that we
+                 * allocate space first */
+                struct i915_vma *vma = i915_gem_obj_to_ggtt(batch_obj);
+                BUG_ON(!vma);
+                vma->bind_vma(vma, batch_obj->cache_level, GLOBAL_BIND);
+        }
+
+        if (flags & I915_DISPATCH_SECURE)
+                exec_start += i915_gem_obj_ggtt_offset(batch_obj);
+        else
+                exec_start += i915_gem_obj_offset(batch_obj, vm);
 
         ret = i915_gem_execbuffer_move_to_gpu(ring, &eb->vmas);
         if (ret)
                 goto err;
 
-        ret = i915_switch_context(ring, file, ctx_id);
+        ret = i915_switch_context(ring, file, ctx);
         if (ret)
                 goto err;
 
@@ -1219,8 +1246,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
                         goto err;
         }
 
-        exec_start = i915_gem_obj_offset(batch_obj, vm) +
-                args->batch_start_offset;
+
         exec_len = args->batch_len;
         if (cliprects) {
                 for (i = 0; i < args->num_cliprects; i++) {
@@ -1249,6 +1275,8 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
         i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj);
 
 err:
+        /* the request owns the ref now */
+        i915_gem_context_unreference(ctx);
         eb_destroy(eb);
 
         mutex_unlock(&dev->struct_mutex);
@@ -1270,7 +1298,6 @@ int
 i915_gem_execbuffer(struct drm_device *dev, void *data,
                     struct drm_file *file)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_execbuffer *args = data;
         struct drm_i915_gem_execbuffer2 exec2;
         struct drm_i915_gem_exec_object *exec_list = NULL;
@@ -1326,8 +1353,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
         exec2.flags = I915_EXEC_RENDER;
         i915_execbuffer2_set_context_id(exec2, 0);
 
-        ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list,
-                                     &dev_priv->gtt.base);
+        ret = i915_gem_do_execbuffer(dev, data, file, &exec2, exec2_list);
         if (!ret) {
                 /* Copy the new buffer offsets back to the user's exec list. */
                 for (i = 0; i < args->buffer_count; i++)
@@ -1353,7 +1379,6 @@ int
 i915_gem_execbuffer2(struct drm_device *dev, void *data,
                      struct drm_file *file)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_execbuffer2 *args = data;
         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
         int ret;
@@ -1384,8 +1409,7 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
                 return -EFAULT;
         }
 
-        ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list,
-                                     &dev_priv->gtt.base);
+        ret = i915_gem_do_execbuffer(dev, data, file, args, exec2_list);
         if (!ret) {
                 /* Copy the new buffer offsets back to the user's exec list. */
                 ret = copy_to_user(to_user_ptr(args->buffers_ptr),
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index d278be110805..ab5e93c30aa2 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -1,5 +1,6 @@
 /*
  * Copyright © 2010 Daniel Vetter
+ * Copyright © 2011-2014 Intel Corporation
  *
  * Permission is hereby granted, free of charge, to any person obtaining a
  * copy of this software and associated documentation files (the "Software"),
@@ -22,12 +23,38 @@
  *
  */
 
+#include <linux/seq_file.h>
 #include <drm/drmP.h>
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
 
+static void gen8_setup_private_ppat(struct drm_i915_private *dev_priv);
+
+bool intel_enable_ppgtt(struct drm_device *dev, bool full)
+{
+        if (i915.enable_ppgtt == 0 || !HAS_ALIASING_PPGTT(dev))
+                return false;
+
+        if (i915.enable_ppgtt == 1 && full)
+                return false;
+
+#ifdef CONFIG_INTEL_IOMMU
+        /* Disable ppgtt on SNB if VT-d is on. */
+        if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
+                DRM_INFO("Disabling PPGTT because VT-d is on\n");
+                return false;
+        }
+#endif
+
+        /* Full ppgtt disabled by default for now due to issues. */
+        if (full)
+                return false; /* HAS_PPGTT(dev) */
+        else
+                return HAS_ALIASING_PPGTT(dev);
+}
+
 #define GEN6_PPGTT_PD_ENTRIES 512
 #define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t))
 typedef uint64_t gen8_gtt_pte_t;
@@ -63,13 +90,31 @@ typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;
 
 #define GEN8_PTES_PER_PAGE              (PAGE_SIZE / sizeof(gen8_gtt_pte_t))
 #define GEN8_PDES_PER_PAGE              (PAGE_SIZE / sizeof(gen8_ppgtt_pde_t))
-#define GEN8_LEGACY_PDPS                4
+
+/* GEN8 legacy style addressis defined as a 3 level page table:
+ * 31:30 | 29:21 | 20:12 |  11:0
+ * PDPE  |  PDE  |  PTE  | offset
+ * The difference as compared to normal x86 3 level page table is the PDPEs are
+ * programmed via register.
+ */
+#define GEN8_PDPE_SHIFT                 30
+#define GEN8_PDPE_MASK                  0x3
+#define GEN8_PDE_SHIFT                  21
+#define GEN8_PDE_MASK                   0x1ff
+#define GEN8_PTE_SHIFT                  12
+#define GEN8_PTE_MASK                   0x1ff
 
 #define PPAT_UNCACHED_INDEX             (_PAGE_PWT | _PAGE_PCD)
 #define PPAT_CACHED_PDE_INDEX           0 /* WB LLC */
 #define PPAT_CACHED_INDEX               _PAGE_PAT /* WB LLCeLLC */
 #define PPAT_DISPLAY_ELLC_INDEX         _PAGE_PCD /* WT eLLC */
 
+static void ppgtt_bind_vma(struct i915_vma *vma,
+                           enum i915_cache_level cache_level,
+                           u32 flags);
+static void ppgtt_unbind_vma(struct i915_vma *vma);
+static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt);
+
 static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
                                              enum i915_cache_level level,
                                              bool valid)
@@ -199,12 +244,19 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
 
 /* Broadwell Page Directory Pointer Descriptors */
 static int gen8_write_pdp(struct intel_ring_buffer *ring, unsigned entry,
-                           uint64_t val)
+                           uint64_t val, bool synchronous)
 {
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
         int ret;
 
         BUG_ON(entry >= 4);
 
+        if (synchronous) {
+                I915_WRITE(GEN8_RING_PDP_UDW(ring, entry), val >> 32);
+                I915_WRITE(GEN8_RING_PDP_LDW(ring, entry), (u32)val);
+                return 0;
+        }
+
         ret = intel_ring_begin(ring, 6);
         if (ret)
                 return ret;
@@ -220,216 +272,357 @@ static int gen8_write_pdp(struct intel_ring_buffer *ring, unsigned entry,
         return 0;
 }
 
-static int gen8_ppgtt_enable(struct drm_device *dev)
+static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
+                          struct intel_ring_buffer *ring,
+                          bool synchronous)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_ring_buffer *ring;
-        struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
-        int i, j, ret;
+        int i, ret;
 
         /* bit of a hack to find the actual last used pd */
         int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE;
 
-        for_each_ring(ring, dev_priv, j) {
-                I915_WRITE(RING_MODE_GEN7(ring),
-                           _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
-        }
-
         for (i = used_pd - 1; i >= 0; i--) {
                 dma_addr_t addr = ppgtt->pd_dma_addr[i];
-                for_each_ring(ring, dev_priv, j) {
-                        ret = gen8_write_pdp(ring, i, addr);
-                        if (ret)
-                                goto err_out;
-                }
+                ret = gen8_write_pdp(ring, i, addr, synchronous);
+                if (ret)
+                        return ret;
         }
-        return 0;
 
-err_out:
-        for_each_ring(ring, dev_priv, j)
-                I915_WRITE(RING_MODE_GEN7(ring),
-                           _MASKED_BIT_DISABLE(GFX_PPGTT_ENABLE));
-        return ret;
+        return 0;
 }
 
 static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
-                                   unsigned first_entry,
-                                   unsigned num_entries,
+                                   uint64_t start,
+                                   uint64_t length,
                                    bool use_scratch)
 {
         struct i915_hw_ppgtt *ppgtt =
                 container_of(vm, struct i915_hw_ppgtt, base);
         gen8_gtt_pte_t *pt_vaddr, scratch_pte;
-        unsigned act_pt = first_entry / GEN8_PTES_PER_PAGE;
-        unsigned first_pte = first_entry % GEN8_PTES_PER_PAGE;
+        unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
+        unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
+        unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
+        unsigned num_entries = length >> PAGE_SHIFT;
         unsigned last_pte, i;
 
         scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,
                                       I915_CACHE_LLC, use_scratch);
 
         while (num_entries) {
-                struct page *page_table = &ppgtt->gen8_pt_pages[act_pt];
+                struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde];
 
-                last_pte = first_pte + num_entries;
+                last_pte = pte + num_entries;
                 if (last_pte > GEN8_PTES_PER_PAGE)
                         last_pte = GEN8_PTES_PER_PAGE;
 
                 pt_vaddr = kmap_atomic(page_table);
 
-                for (i = first_pte; i < last_pte; i++)
+                for (i = pte; i < last_pte; i++) {
                         pt_vaddr[i] = scratch_pte;
+                        num_entries--;
+                }
 
                 kunmap_atomic(pt_vaddr);
 
-                num_entries -= last_pte - first_pte;
-                first_pte = 0;
-                act_pt++;
+                pte = 0;
+                if (++pde == GEN8_PDES_PER_PAGE) {
+                        pdpe++;
+                        pde = 0;
+                }
         }
 }
 
 static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
                                       struct sg_table *pages,
-                                      unsigned first_entry,
+                                      uint64_t start,
                                       enum i915_cache_level cache_level)
 {
         struct i915_hw_ppgtt *ppgtt =
                 container_of(vm, struct i915_hw_ppgtt, base);
         gen8_gtt_pte_t *pt_vaddr;
-        unsigned act_pt = first_entry / GEN8_PTES_PER_PAGE;
-        unsigned act_pte = first_entry % GEN8_PTES_PER_PAGE;
+        unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
+        unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
+        unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
         struct sg_page_iter sg_iter;
 
         pt_vaddr = NULL;
+
         for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
+                if (WARN_ON(pdpe >= GEN8_LEGACY_PDPS))
+                        break;
+
                 if (pt_vaddr == NULL)
-                        pt_vaddr = kmap_atomic(&ppgtt->gen8_pt_pages[act_pt]);
+                        pt_vaddr = kmap_atomic(ppgtt->gen8_pt_pages[pdpe][pde]);
 
-                pt_vaddr[act_pte] =
+                pt_vaddr[pte] =
                         gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
                                         cache_level, true);
-                if (++act_pte == GEN8_PTES_PER_PAGE) {
+                if (++pte == GEN8_PTES_PER_PAGE) {
                         kunmap_atomic(pt_vaddr);
                         pt_vaddr = NULL;
-                        act_pt++;
-                        act_pte = 0;
+                        if (++pde == GEN8_PDES_PER_PAGE) {
+                                pdpe++;
+                                pde = 0;
+                        }
+                        pte = 0;
                 }
         }
         if (pt_vaddr)
                 kunmap_atomic(pt_vaddr);
 }
 
+static void gen8_free_page_tables(struct page **pt_pages)
+{
+        int i;
+
+        if (pt_pages == NULL)
+                return;
+
+        for (i = 0; i < GEN8_PDES_PER_PAGE; i++)
+                if (pt_pages[i])
+                        __free_pages(pt_pages[i], 0);
+}
+
+static void gen8_ppgtt_free(const struct i915_hw_ppgtt *ppgtt)
+{
+        int i;
+
+        for (i = 0; i < ppgtt->num_pd_pages; i++) {
+                gen8_free_page_tables(ppgtt->gen8_pt_pages[i]);
+                kfree(ppgtt->gen8_pt_pages[i]);
+                kfree(ppgtt->gen8_pt_dma_addr[i]);
+        }
+
+        __free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));
+}
+
+static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
+{
+        struct pci_dev *hwdev = ppgtt->base.dev->pdev;
+        int i, j;
+
+        for (i = 0; i < ppgtt->num_pd_pages; i++) {
+                /* TODO: In the future we'll support sparse mappings, so this
+                 * will have to change. */
+                if (!ppgtt->pd_dma_addr[i])
+                        continue;
+
+                pci_unmap_page(hwdev, ppgtt->pd_dma_addr[i], PAGE_SIZE,
+                               PCI_DMA_BIDIRECTIONAL);
+
+                for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
+                        dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
+                        if (addr)
+                                pci_unmap_page(hwdev, addr, PAGE_SIZE,
+                                               PCI_DMA_BIDIRECTIONAL);
+                }
+        }
+}
+
 static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 {
         struct i915_hw_ppgtt *ppgtt =
                 container_of(vm, struct i915_hw_ppgtt, base);
-        int i, j;
 
+        list_del(&vm->global_link);
         drm_mm_takedown(&vm->mm);
 
-        for (i = 0; i < ppgtt->num_pd_pages ; i++) {
-                if (ppgtt->pd_dma_addr[i]) {
-                        pci_unmap_page(ppgtt->base.dev->pdev,
-                                       ppgtt->pd_dma_addr[i],
-                                       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+        gen8_ppgtt_unmap_pages(ppgtt);
+        gen8_ppgtt_free(ppgtt);
+}
 
-                        for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
-                                dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
-                                if (addr)
-                                        pci_unmap_page(ppgtt->base.dev->pdev,
-                                                       addr,
-                                                       PAGE_SIZE,
-                                                       PCI_DMA_BIDIRECTIONAL);
+static struct page **__gen8_alloc_page_tables(void)
+{
+        struct page **pt_pages;
+        int i;
 
-                        }
-                }
-                kfree(ppgtt->gen8_pt_dma_addr[i]);
+        pt_pages = kcalloc(GEN8_PDES_PER_PAGE, sizeof(struct page *), GFP_KERNEL);
+        if (!pt_pages)
+                return ERR_PTR(-ENOMEM);
+
+        for (i = 0; i < GEN8_PDES_PER_PAGE; i++) {
+                pt_pages[i] = alloc_page(GFP_KERNEL);
+                if (!pt_pages[i])
+                        goto bail;
         }
 
-        __free_pages(ppgtt->gen8_pt_pages, get_order(ppgtt->num_pt_pages << PAGE_SHIFT));
-        __free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));
+        return pt_pages;
+
+bail:
+        gen8_free_page_tables(pt_pages);
+        kfree(pt_pages);
+        return ERR_PTR(-ENOMEM);
 }
 
-/**
- * GEN8 legacy ppgtt programming is accomplished through 4 PDP registers with a
- * net effect resembling a 2-level page table in normal x86 terms. Each PDP
- * represents 1GB of memory
- * 4 * 512 * 512 * 4096 = 4GB legacy 32b address space.
- *
- * TODO: Do something with the size parameter
- **/
-static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
+static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt,
+                                           const int max_pdp)
 {
-        struct page *pt_pages;
-        int i, j, ret = -ENOMEM;
-        const int max_pdp = DIV_ROUND_UP(size, 1 << 30);
-        const int num_pt_pages = GEN8_PDES_PER_PAGE * max_pdp;
+        struct page **pt_pages[GEN8_LEGACY_PDPS];
+        int i, ret;
 
-        if (size % (1<<30))
-                DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size);
+        for (i = 0; i < max_pdp; i++) {
+                pt_pages[i] = __gen8_alloc_page_tables();
+                if (IS_ERR(pt_pages[i])) {
+                        ret = PTR_ERR(pt_pages[i]);
+                        goto unwind_out;
+                }
+        }
 
-        /* FIXME: split allocation into smaller pieces. For now we only ever do
-         * this once, but with full PPGTT, the multiple contiguous allocations
-         * will be bad.
+        /* NB: Avoid touching gen8_pt_pages until last to keep the allocation,
+         * "atomic" - for cleanup purposes.
          */
+        for (i = 0; i < max_pdp; i++)
+                ppgtt->gen8_pt_pages[i] = pt_pages[i];
+
+        return 0;
+
+unwind_out:
+        while (i--) {
+                gen8_free_page_tables(pt_pages[i]);
+                kfree(pt_pages[i]);
+        }
+
+        return ret;
+}
+
+static int gen8_ppgtt_allocate_dma(struct i915_hw_ppgtt *ppgtt)
+{
+        int i;
+
+        for (i = 0; i < ppgtt->num_pd_pages; i++) {
+                ppgtt->gen8_pt_dma_addr[i] = kcalloc(GEN8_PDES_PER_PAGE,
+                                                     sizeof(dma_addr_t),
+                                                     GFP_KERNEL);
+                if (!ppgtt->gen8_pt_dma_addr[i])
+                        return -ENOMEM;
+        }
+
+        return 0;
+}
+
+static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
+                                                const int max_pdp)
+{
         ppgtt->pd_pages = alloc_pages(GFP_KERNEL, get_order(max_pdp << PAGE_SHIFT));
         if (!ppgtt->pd_pages)
                 return -ENOMEM;
 
-        pt_pages = alloc_pages(GFP_KERNEL, get_order(num_pt_pages << PAGE_SHIFT));
-        if (!pt_pages) {
+        ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT);
+        BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS);
+
+        return 0;
+}
+
+static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
+                            const int max_pdp)
+{
+        int ret;
+
+        ret = gen8_ppgtt_allocate_page_directories(ppgtt, max_pdp);
+        if (ret)
+                return ret;
+
+        ret = gen8_ppgtt_allocate_page_tables(ppgtt, max_pdp);
+        if (ret) {
                 __free_pages(ppgtt->pd_pages, get_order(max_pdp << PAGE_SHIFT));
-                return -ENOMEM;
+                return ret;
         }
 
-        ppgtt->gen8_pt_pages = pt_pages;
-        ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT);
-        ppgtt->num_pt_pages = 1 << get_order(num_pt_pages << PAGE_SHIFT);
         ppgtt->num_pd_entries = max_pdp * GEN8_PDES_PER_PAGE;
-        ppgtt->enable = gen8_ppgtt_enable;
-        ppgtt->base.clear_range = gen8_ppgtt_clear_range;
-        ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
-        ppgtt->base.cleanup = gen8_ppgtt_cleanup;
-        ppgtt->base.start = 0;
-        ppgtt->base.total = ppgtt->num_pt_pages * GEN8_PTES_PER_PAGE * PAGE_SIZE;
 
-        BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS);
+        ret = gen8_ppgtt_allocate_dma(ppgtt);
+        if (ret)
+                gen8_ppgtt_free(ppgtt);
 
-        /*
-         * - Create a mapping for the page directories.
-         * - For each page directory:
-         *      allocate space for page table mappings.
-         *      map each page table
-         */
-        for (i = 0; i < max_pdp; i++) {
-                dma_addr_t temp;
-                temp = pci_map_page(ppgtt->base.dev->pdev,
-                                    &ppgtt->pd_pages[i], 0,
-                                    PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
-                if (pci_dma_mapping_error(ppgtt->base.dev->pdev, temp))
-                        goto err_out;
+        return ret;
+}
 
-                ppgtt->pd_dma_addr[i] = temp;
+static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
+                                             const int pd)
+{
+        dma_addr_t pd_addr;
+        int ret;
 
-                ppgtt->gen8_pt_dma_addr[i] = kmalloc(sizeof(dma_addr_t) * GEN8_PDES_PER_PAGE, GFP_KERNEL);
-                if (!ppgtt->gen8_pt_dma_addr[i])
-                        goto err_out;
+        pd_addr = pci_map_page(ppgtt->base.dev->pdev,
+                               &ppgtt->pd_pages[pd], 0,
+                               PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
 
-                for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
-                        struct page *p = &pt_pages[i * GEN8_PDES_PER_PAGE + j];
-                        temp = pci_map_page(ppgtt->base.dev->pdev,
-                                            p, 0, PAGE_SIZE,
-                                            PCI_DMA_BIDIRECTIONAL);
+        ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr);
+        if (ret)
+                return ret;
 
-                        if (pci_dma_mapping_error(ppgtt->base.dev->pdev, temp))
-                                goto err_out;
+        ppgtt->pd_dma_addr[pd] = pd_addr;
 
-                        ppgtt->gen8_pt_dma_addr[i][j] = temp;
+        return 0;
+}
+
+static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,
+                                        const int pd,
+                                        const int pt)
+{
+        dma_addr_t pt_addr;
+        struct page *p;
+        int ret;
+
+        p = ppgtt->gen8_pt_pages[pd][pt];
+        pt_addr = pci_map_page(ppgtt->base.dev->pdev,
+                               p, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+        ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr);
+        if (ret)
+                return ret;
+
+        ppgtt->gen8_pt_dma_addr[pd][pt] = pt_addr;
+
+        return 0;
+}
+
+/**
+ * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
+ * with a net effect resembling a 2-level page table in normal x86 terms. Each
+ * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
+ * space.
+ *
+ * FIXME: split allocation into smaller pieces. For now we only ever do this
+ * once, but with full PPGTT, the multiple contiguous allocations will be bad.
+ * TODO: Do something with the size parameter
+ */
+static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
+{
+        const int max_pdp = DIV_ROUND_UP(size, 1 << 30);
+        const int min_pt_pages = GEN8_PDES_PER_PAGE * max_pdp;
+        int i, j, ret;
+
+        if (size % (1<<30))
+                DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size);
+
+        /* 1. Do all our allocations for page directories and page tables. */
+        ret = gen8_ppgtt_alloc(ppgtt, max_pdp);
+        if (ret)
+                return ret;
+
+        /*
+         * 2. Create DMA mappings for the page directories and page tables.
+         */
+        for (i = 0; i < max_pdp; i++) {
+                ret = gen8_ppgtt_setup_page_directories(ppgtt, i);
+                if (ret)
+                        goto bail;
+
+                for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
+                        ret = gen8_ppgtt_setup_page_tables(ppgtt, i, j);
+                        if (ret)
+                                goto bail;
                 }
         }
 
-        /* For now, the PPGTT helper functions all require that the PDEs are
+        /*
+         * 3. Map all the page directory entires to point to the page tables
+         * we've allocated.
+         *
+         * For now, the PPGTT helper functions all require that the PDEs are
          * plugged in correctly. So we do that now/here. For aliasing PPGTT, we
-         * will never need to touch the PDEs again */
+         * will never need to touch the PDEs again.
+         */
         for (i = 0; i < max_pdp; i++) {
                 gen8_ppgtt_pde_t *pd_vaddr;
                 pd_vaddr = kmap_atomic(&ppgtt->pd_pages[i]);
@@ -441,23 +634,85 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
                 kunmap_atomic(pd_vaddr);
         }
 
-        ppgtt->base.clear_range(&ppgtt->base, 0,
-                                ppgtt->num_pd_entries * GEN8_PTES_PER_PAGE,
-                                true);
+        ppgtt->enable = gen8_ppgtt_enable;
+        ppgtt->switch_mm = gen8_mm_switch;
+        ppgtt->base.clear_range = gen8_ppgtt_clear_range;
+        ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
+        ppgtt->base.cleanup = gen8_ppgtt_cleanup;
+        ppgtt->base.start = 0;
+        ppgtt->base.total = ppgtt->num_pd_entries * GEN8_PTES_PER_PAGE * PAGE_SIZE;
+
+        ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
 
         DRM_DEBUG_DRIVER("Allocated %d pages for page directories (%d wasted)\n",
                          ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);
         DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%lld wasted)\n",
-                         ppgtt->num_pt_pages,
-                         (ppgtt->num_pt_pages - num_pt_pages) +
-                         size % (1<<30));
+                         ppgtt->num_pd_entries,
+                         (ppgtt->num_pd_entries - min_pt_pages) + size % (1<<30));
         return 0;
 
-err_out:
-        ppgtt->base.cleanup(&ppgtt->base);
+bail:
+        gen8_ppgtt_unmap_pages(ppgtt);
+        gen8_ppgtt_free(ppgtt);
         return ret;
 }
 
+static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
+{
+        struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
+        struct i915_address_space *vm = &ppgtt->base;
+        gen6_gtt_pte_t __iomem *pd_addr;
+        gen6_gtt_pte_t scratch_pte;
+        uint32_t pd_entry;
+        int pte, pde;
+
+        scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true);
+
+        pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm +
+                ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
+
+        seq_printf(m, "  VM %p (pd_offset %x-%x):\n", vm,
+                   ppgtt->pd_offset, ppgtt->pd_offset + ppgtt->num_pd_entries);
+        for (pde = 0; pde < ppgtt->num_pd_entries; pde++) {
+                u32 expected;
+                gen6_gtt_pte_t *pt_vaddr;
+                dma_addr_t pt_addr = ppgtt->pt_dma_addr[pde];
+                pd_entry = readl(pd_addr + pde);
+                expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
+
+                if (pd_entry != expected)
+                        seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
+                                   pde,
+                                   pd_entry,
+                                   expected);
+                seq_printf(m, "\tPDE: %x\n", pd_entry);
+
+                pt_vaddr = kmap_atomic(ppgtt->pt_pages[pde]);
+                for (pte = 0; pte < I915_PPGTT_PT_ENTRIES; pte+=4) {
+                        unsigned long va =
+                                (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
+                                (pte * PAGE_SIZE);
+                        int i;
+                        bool found = false;
+                        for (i = 0; i < 4; i++)
+                                if (pt_vaddr[pte + i] != scratch_pte)
+                                        found = true;
+                        if (!found)
+                                continue;
+
+                        seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
+                        for (i = 0; i < 4; i++) {
+                                if (pt_vaddr[pte + i] != scratch_pte)
+                                        seq_printf(m, " %08x", pt_vaddr[pte + i]);
+                                else
+                                        seq_puts(m, "  SCRATCH ");
+                        }
+                        seq_puts(m, "\n");
+                }
+                kunmap_atomic(pt_vaddr);
+        }
+}
+
 static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
 {
         struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
@@ -480,73 +735,235 @@ static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
         readl(pd_addr);
 }
 
-static int gen6_ppgtt_enable(struct drm_device *dev)
+static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
+{
+        BUG_ON(ppgtt->pd_offset & 0x3f);
+
+        return (ppgtt->pd_offset / 64) << 16;
+}
+
+static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
+                         struct intel_ring_buffer *ring,
+                         bool synchronous)
+{
+        struct drm_device *dev = ppgtt->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
+
+        /* If we're in reset, we can assume the GPU is sufficiently idle to
+         * manually frob these bits. Ideally we could use the ring functions,
+         * except our error handling makes it quite difficult (can't use
+         * intel_ring_begin, ring->flush, or intel_ring_advance)
+         *
+         * FIXME: We should try not to special case reset
+         */
+        if (synchronous ||
+            i915_reset_in_progress(&dev_priv->gpu_error)) {
+                WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt);
+                I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
+                I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
+                POSTING_READ(RING_PP_DIR_BASE(ring));
+                return 0;
+        }
+
+        /* NB: TLBs must be flushed and invalidated before a switch */
+        ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
+        if (ret)
+                return ret;
+
+        ret = intel_ring_begin(ring, 6);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
+        intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
+        intel_ring_emit(ring, PP_DIR_DCLV_2G);
+        intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
+        intel_ring_emit(ring, get_pd_offset(ppgtt));
+        intel_ring_emit(ring, MI_NOOP);
+        intel_ring_advance(ring);
+
+        return 0;
+}
+
+static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
+                          struct intel_ring_buffer *ring,
+                          bool synchronous)
+{
+        struct drm_device *dev = ppgtt->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
+
+        /* If we're in reset, we can assume the GPU is sufficiently idle to
+         * manually frob these bits. Ideally we could use the ring functions,
+         * except our error handling makes it quite difficult (can't use
+         * intel_ring_begin, ring->flush, or intel_ring_advance)
+         *
+         * FIXME: We should try not to special case reset
+         */
+        if (synchronous ||
+            i915_reset_in_progress(&dev_priv->gpu_error)) {
+                WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt);
+                I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
+                I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
+                POSTING_READ(RING_PP_DIR_BASE(ring));
+                return 0;
+        }
+
+        /* NB: TLBs must be flushed and invalidated before a switch */
+        ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
+        if (ret)
+                return ret;
+
+        ret = intel_ring_begin(ring, 6);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
+        intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
+        intel_ring_emit(ring, PP_DIR_DCLV_2G);
+        intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
+        intel_ring_emit(ring, get_pd_offset(ppgtt));
+        intel_ring_emit(ring, MI_NOOP);
+        intel_ring_advance(ring);
+
+        /* XXX: RCS is the only one to auto invalidate the TLBs? */
+        if (ring->id != RCS) {
+                ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
+static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
+                          struct intel_ring_buffer *ring,
+                          bool synchronous)
+{
+        struct drm_device *dev = ppgtt->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (!synchronous)
+                return 0;
+
+        I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
+        I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
+
+        POSTING_READ(RING_PP_DIR_DCLV(ring));
+
+        return 0;
+}
+
+static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
-        uint32_t pd_offset;
+        struct drm_device *dev = ppgtt->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring;
-        struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
-        int i;
+        int j, ret;
 
-        BUG_ON(ppgtt->pd_offset & 0x3f);
+        for_each_ring(ring, dev_priv, j) {
+                I915_WRITE(RING_MODE_GEN7(ring),
+                           _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 
-        gen6_write_pdes(ppgtt);
+                /* We promise to do a switch later with FULL PPGTT. If this is
+                 * aliasing, this is the one and only switch we'll do */
+                if (USES_FULL_PPGTT(dev))
+                        continue;
 
-        pd_offset = ppgtt->pd_offset;
-        pd_offset /= 64; /* in cachelines, */
-        pd_offset <<= 16;
+                ret = ppgtt->switch_mm(ppgtt, ring, true);
+                if (ret)
+                        goto err_out;
+        }
 
-        if (INTEL_INFO(dev)->gen == 6) {
-                uint32_t ecochk, gab_ctl, ecobits;
+        return 0;
 
-                ecobits = I915_READ(GAC_ECO_BITS);
-                I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
-                                         ECOBITS_PPGTT_CACHE64B);
+err_out:
+        for_each_ring(ring, dev_priv, j)
+                I915_WRITE(RING_MODE_GEN7(ring),
+                           _MASKED_BIT_DISABLE(GFX_PPGTT_ENABLE));
+        return ret;
+}
 
-                gab_ctl = I915_READ(GAB_CTL);
-                I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
+static int gen7_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
+{
+        struct drm_device *dev = ppgtt->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring;
+        uint32_t ecochk, ecobits;
+        int i;
 
-                ecochk = I915_READ(GAM_ECOCHK);
-                I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
-                                       ECOCHK_PPGTT_CACHE64B);
-                I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
-        } else if (INTEL_INFO(dev)->gen >= 7) {
-                uint32_t ecochk, ecobits;
+        ecobits = I915_READ(GAC_ECO_BITS);
+        I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
 
-                ecobits = I915_READ(GAC_ECO_BITS);
-                I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
+        ecochk = I915_READ(GAM_ECOCHK);
+        if (IS_HASWELL(dev)) {
+                ecochk |= ECOCHK_PPGTT_WB_HSW;
+        } else {
+                ecochk |= ECOCHK_PPGTT_LLC_IVB;
+                ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
+        }
+        I915_WRITE(GAM_ECOCHK, ecochk);
 
-                ecochk = I915_READ(GAM_ECOCHK);
-                if (IS_HASWELL(dev)) {
-                        ecochk |= ECOCHK_PPGTT_WB_HSW;
-                } else {
-                        ecochk |= ECOCHK_PPGTT_LLC_IVB;
-                        ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
-                }
-                I915_WRITE(GAM_ECOCHK, ecochk);
+        for_each_ring(ring, dev_priv, i) {
+                int ret;
                 /* GFX_MODE is per-ring on gen7+ */
+                I915_WRITE(RING_MODE_GEN7(ring),
+                           _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
+
+                /* We promise to do a switch later with FULL PPGTT. If this is
+                 * aliasing, this is the one and only switch we'll do */
+                if (USES_FULL_PPGTT(dev))
+                        continue;
+
+                ret = ppgtt->switch_mm(ppgtt, ring, true);
+                if (ret)
+                        return ret;
         }
 
-        for_each_ring(ring, dev_priv, i) {
-                if (INTEL_INFO(dev)->gen >= 7)
-                        I915_WRITE(RING_MODE_GEN7(ring),
-                                   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
+        return 0;
+}
 
-                I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
-                I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
+static int gen6_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
+{
+        struct drm_device *dev = ppgtt->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_ring_buffer *ring;
+        uint32_t ecochk, gab_ctl, ecobits;
+        int i;
+
+        ecobits = I915_READ(GAC_ECO_BITS);
+        I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
+                   ECOBITS_PPGTT_CACHE64B);
+
+        gab_ctl = I915_READ(GAB_CTL);
+        I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
+
+        ecochk = I915_READ(GAM_ECOCHK);
+        I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
+
+        I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
+
+        for_each_ring(ring, dev_priv, i) {
+                int ret = ppgtt->switch_mm(ppgtt, ring, true);
+                if (ret)
+                        return ret;
         }
+
         return 0;
 }
 
 /* PPGTT support for Sandybdrige/Gen6 and later */
 static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
-                                   unsigned first_entry,
-                                   unsigned num_entries,
+                                   uint64_t start,
+                                   uint64_t length,
                                    bool use_scratch)
 {
         struct i915_hw_ppgtt *ppgtt =
                 container_of(vm, struct i915_hw_ppgtt, base);
         gen6_gtt_pte_t *pt_vaddr, scratch_pte;
+        unsigned first_entry = start >> PAGE_SHIFT;
+        unsigned num_entries = length >> PAGE_SHIFT;
         unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
         unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
         unsigned last_pte, i;
@@ -573,12 +990,13 @@ static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
 
 static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
                                       struct sg_table *pages,
-                                      unsigned first_entry,
+                                      uint64_t start,
                                       enum i915_cache_level cache_level)
 {
         struct i915_hw_ppgtt *ppgtt =
                 container_of(vm, struct i915_hw_ppgtt, base);
         gen6_gtt_pte_t *pt_vaddr;
+        unsigned first_entry = start >> PAGE_SHIFT;
         unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
         unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
         struct sg_page_iter sg_iter;
@@ -602,65 +1020,130 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
                 kunmap_atomic(pt_vaddr);
 }
 
-static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
+static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
 {
-        struct i915_hw_ppgtt *ppgtt =
-                container_of(vm, struct i915_hw_ppgtt, base);
         int i;
 
-        drm_mm_takedown(&ppgtt->base.mm);
-
         if (ppgtt->pt_dma_addr) {
                 for (i = 0; i < ppgtt->num_pd_entries; i++)
                         pci_unmap_page(ppgtt->base.dev->pdev,
                                        ppgtt->pt_dma_addr[i],
                                        4096, PCI_DMA_BIDIRECTIONAL);
         }
+}
+
+static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
+{
+        int i;
 
         kfree(ppgtt->pt_dma_addr);
         for (i = 0; i < ppgtt->num_pd_entries; i++)
                 __free_page(ppgtt->pt_pages[i]);
         kfree(ppgtt->pt_pages);
-        kfree(ppgtt);
 }
 
-static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
+static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 {
+        struct i915_hw_ppgtt *ppgtt =
+                container_of(vm, struct i915_hw_ppgtt, base);
+
+        list_del(&vm->global_link);
+        drm_mm_takedown(&ppgtt->base.mm);
+        drm_mm_remove_node(&ppgtt->node);
+
+        gen6_ppgtt_unmap_pages(ppgtt);
+        gen6_ppgtt_free(ppgtt);
+}
+
+static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
+{
+#define GEN6_PD_ALIGN (PAGE_SIZE * 16)
+#define GEN6_PD_SIZE (GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE)
         struct drm_device *dev = ppgtt->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned first_pd_entry_in_global_pt;
-        int i;
-        int ret = -ENOMEM;
+        bool retried = false;
+        int ret;
 
-        /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
-         * entries. For aliasing ppgtt support we just steal them at the end for
-         * now. */
-        first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
+        /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
+         * allocator works in address space sizes, so it's multiplied by page
+         * size. We allocate at the top of the GTT to avoid fragmentation.
+         */
+        BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
+alloc:
+        ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
+                                                  &ppgtt->node, GEN6_PD_SIZE,
+                                                  GEN6_PD_ALIGN, 0,
+                                                  0, dev_priv->gtt.base.total,
+                                                  DRM_MM_SEARCH_DEFAULT,
+                                                  DRM_MM_CREATE_DEFAULT);
+        if (ret == -ENOSPC && !retried) {
+                ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,
+                                               GEN6_PD_SIZE, GEN6_PD_ALIGN,
+                                               I915_CACHE_NONE, 0);
+                if (ret)
+                        return ret;
+
+                retried = true;
+                goto alloc;
+        }
+
+        if (ppgtt->node.start < dev_priv->gtt.mappable_end)
+                DRM_DEBUG("Forced to use aperture for PDEs\n");
 
-        ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
         ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
-        ppgtt->enable = gen6_ppgtt_enable;
-        ppgtt->base.clear_range = gen6_ppgtt_clear_range;
-        ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
-        ppgtt->base.cleanup = gen6_ppgtt_cleanup;
-        ppgtt->base.scratch = dev_priv->gtt.base.scratch;
-        ppgtt->base.start = 0;
-        ppgtt->base.total = GEN6_PPGTT_PD_ENTRIES * I915_PPGTT_PT_ENTRIES * PAGE_SIZE;
+        return ret;
+}
+
+static int gen6_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
+{
+        int i;
+
         ppgtt->pt_pages = kcalloc(ppgtt->num_pd_entries, sizeof(struct page *),
                                   GFP_KERNEL);
+
         if (!ppgtt->pt_pages)
                 return -ENOMEM;
 
         for (i = 0; i < ppgtt->num_pd_entries; i++) {
                 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
-                if (!ppgtt->pt_pages[i])
-                        goto err_pt_alloc;
+                if (!ppgtt->pt_pages[i]) {
+                        gen6_ppgtt_free(ppgtt);
+                        return -ENOMEM;
+                }
+        }
+
+        return 0;
+}
+
+static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
+{
+        int ret;
+
+        ret = gen6_ppgtt_allocate_page_directories(ppgtt);
+        if (ret)
+                return ret;
+
+        ret = gen6_ppgtt_allocate_page_tables(ppgtt);
+        if (ret) {
+                drm_mm_remove_node(&ppgtt->node);
+                return ret;
         }
 
         ppgtt->pt_dma_addr = kcalloc(ppgtt->num_pd_entries, sizeof(dma_addr_t),
                                      GFP_KERNEL);
-        if (!ppgtt->pt_dma_addr)
-                goto err_pt_alloc;
+        if (!ppgtt->pt_dma_addr) {
+                drm_mm_remove_node(&ppgtt->node);
+                gen6_ppgtt_free(ppgtt);
+                return -ENOMEM;
+        }
+
+        return 0;
+}
+
+static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)
+{
+        struct drm_device *dev = ppgtt->base.dev;
+        int i;
 
         for (i = 0; i < ppgtt->num_pd_entries; i++) {
                 dma_addr_t pt_addr;
@@ -669,48 +1152,71 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
                                        PCI_DMA_BIDIRECTIONAL);
 
                 if (pci_dma_mapping_error(dev->pdev, pt_addr)) {
-                        ret = -EIO;
-                        goto err_pd_pin;
-
+                        gen6_ppgtt_unmap_pages(ppgtt);
+                        return -EIO;
                 }
+
                 ppgtt->pt_dma_addr[i] = pt_addr;
         }
 
-        ppgtt->base.clear_range(&ppgtt->base, 0,
-                                ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES, true);
+        return 0;
+}
 
-        ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);
+static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
+{
+        struct drm_device *dev = ppgtt->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
 
-        return 0;
+        ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
+        if (IS_GEN6(dev)) {
+                ppgtt->enable = gen6_ppgtt_enable;
+                ppgtt->switch_mm = gen6_mm_switch;
+        } else if (IS_HASWELL(dev)) {
+                ppgtt->enable = gen7_ppgtt_enable;
+                ppgtt->switch_mm = hsw_mm_switch;
+        } else if (IS_GEN7(dev)) {
+                ppgtt->enable = gen7_ppgtt_enable;
+                ppgtt->switch_mm = gen7_mm_switch;
+        } else
+                BUG();
 
-err_pd_pin:
-        if (ppgtt->pt_dma_addr) {
-                for (i--; i >= 0; i--)
-                        pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
-                                       4096, PCI_DMA_BIDIRECTIONAL);
-        }
-err_pt_alloc:
-        kfree(ppgtt->pt_dma_addr);
-        for (i = 0; i < ppgtt->num_pd_entries; i++) {
-                if (ppgtt->pt_pages[i])
-                        __free_page(ppgtt->pt_pages[i]);
+        ret = gen6_ppgtt_alloc(ppgtt);
+        if (ret)
+                return ret;
+
+        ret = gen6_ppgtt_setup_page_tables(ppgtt);
+        if (ret) {
+                gen6_ppgtt_free(ppgtt);
+                return ret;
         }
-        kfree(ppgtt->pt_pages);
 
-        return ret;
+        ppgtt->base.clear_range = gen6_ppgtt_clear_range;
+        ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
+        ppgtt->base.cleanup = gen6_ppgtt_cleanup;
+        ppgtt->base.start = 0;
+        ppgtt->base.total =  ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES * PAGE_SIZE;
+        ppgtt->debug_dump = gen6_dump_ppgtt;
+
+        ppgtt->pd_offset =
+                ppgtt->node.start / PAGE_SIZE * sizeof(gen6_gtt_pte_t);
+
+        ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
+
+        DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",
+                         ppgtt->node.size >> 20,
+                         ppgtt->node.start / PAGE_SIZE);
+
+        return 0;
 }
 
-static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
+int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct i915_hw_ppgtt *ppgtt;
-        int ret;
-
-        ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
-        if (!ppgtt)
-                return -ENOMEM;
+        int ret = 0;
 
         ppgtt->base.dev = dev;
+        ppgtt->base.scratch = dev_priv->gtt.base.scratch;
 
         if (INTEL_INFO(dev)->gen < 8)
                 ret = gen6_ppgtt_init(ppgtt);
@@ -719,45 +1225,37 @@ static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
         else
                 BUG();
 
-        if (ret)
-                kfree(ppgtt);
-        else {
-                dev_priv->mm.aliasing_ppgtt = ppgtt;
+        if (!ret) {
+                struct drm_i915_private *dev_priv = dev->dev_private;
+                kref_init(&ppgtt->ref);
                 drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
                             ppgtt->base.total);
+                i915_init_vm(dev_priv, &ppgtt->base);
+                if (INTEL_INFO(dev)->gen < 8) {
+                        gen6_write_pdes(ppgtt);
+                        DRM_DEBUG("Adding PPGTT at offset %x\n",
+                                  ppgtt->pd_offset << 10);
+                }
         }
 
         return ret;
 }
 
-void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
+static void
+ppgtt_bind_vma(struct i915_vma *vma,
+               enum i915_cache_level cache_level,
+               u32 flags)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
-
-        if (!ppgtt)
-                return;
-
-        ppgtt->base.cleanup(&ppgtt->base);
-        dev_priv->mm.aliasing_ppgtt = NULL;
+        vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
+                                cache_level);
 }
 
-void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
-                            struct drm_i915_gem_object *obj,
-                            enum i915_cache_level cache_level)
+static void ppgtt_unbind_vma(struct i915_vma *vma)
 {
-        ppgtt->base.insert_entries(&ppgtt->base, obj->pages,
-                                   i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
-                                   cache_level);
-}
-
-void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
-                              struct drm_i915_gem_object *obj)
-{
-        ppgtt->base.clear_range(&ppgtt->base,
-                                i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
-                                obj->base.size >> PAGE_SHIFT,
-                                true);
+        vma->vm->clear_range(vma->vm,
+                             vma->node.start,
+                             vma->obj->base.size,
+                             true);
 }
 
 extern int intel_iommu_gfx_mapped;
@@ -840,8 +1338,8 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
         i915_check_and_clear_faults(dev);
 
         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
-                                       dev_priv->gtt.base.start / PAGE_SIZE,
-                                       dev_priv->gtt.base.total / PAGE_SIZE,
+                                       dev_priv->gtt.base.start,
+                                       dev_priv->gtt.base.total,
                                        true);
 }
 
@@ -849,18 +1347,46 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_object *obj;
+        struct i915_address_space *vm;
 
         i915_check_and_clear_faults(dev);
 
         /* First fill our portion of the GTT with scratch pages */
         dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
-                                       dev_priv->gtt.base.start / PAGE_SIZE,
-                                       dev_priv->gtt.base.total / PAGE_SIZE,
+                                       dev_priv->gtt.base.start,
+                                       dev_priv->gtt.base.total,
                                        true);
 
         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
+                struct i915_vma *vma = i915_gem_obj_to_vma(obj,
+                                                           &dev_priv->gtt.base);
+                if (!vma)
+                        continue;
+
                 i915_gem_clflush_object(obj, obj->pin_display);
-                i915_gem_gtt_bind_object(obj, obj->cache_level);
+                /* The bind_vma code tries to be smart about tracking mappings.
+                 * Unfortunately above, we've just wiped out the mappings
+                 * without telling our object about it. So we need to fake it.
+                 */
+                obj->has_global_gtt_mapping = 0;
+                vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
+        }
+
+
+        if (INTEL_INFO(dev)->gen >= 8) {
+                gen8_setup_private_ppat(dev_priv);
+                return;
+        }
+
+        list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
+                /* TODO: Perhaps it shouldn't be gen6 specific */
+                if (i915_is_ggtt(vm)) {
+                        if (dev_priv->mm.aliasing_ppgtt)
+                                gen6_write_pdes(dev_priv->mm.aliasing_ppgtt);
+                        continue;
+                }
+
+                gen6_write_pdes(container_of(vm, struct i915_hw_ppgtt, base));
         }
 
         i915_gem_chipset_flush(dev);
@@ -891,10 +1417,11 @@ static inline void gen8_set_pte(void __iomem *addr, gen8_gtt_pte_t pte)
 
 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
                                      struct sg_table *st,
-                                     unsigned int first_entry,
+                                     uint64_t start,
                                      enum i915_cache_level level)
 {
         struct drm_i915_private *dev_priv = vm->dev->dev_private;
+        unsigned first_entry = start >> PAGE_SHIFT;
         gen8_gtt_pte_t __iomem *gtt_entries =
                 (gen8_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
         int i = 0;
@@ -936,10 +1463,11 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
  */
 static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
                                      struct sg_table *st,
-                                     unsigned int first_entry,
+                                     uint64_t start,
                                      enum i915_cache_level level)
 {
         struct drm_i915_private *dev_priv = vm->dev->dev_private;
+        unsigned first_entry = start >> PAGE_SHIFT;
         gen6_gtt_pte_t __iomem *gtt_entries =
                 (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
         int i = 0;
@@ -971,11 +1499,13 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
 }
 
 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
-                                  unsigned int first_entry,
-                                  unsigned int num_entries,
+                                  uint64_t start,
+                                  uint64_t length,
                                   bool use_scratch)
 {
         struct drm_i915_private *dev_priv = vm->dev->dev_private;
+        unsigned first_entry = start >> PAGE_SHIFT;
+        unsigned num_entries = length >> PAGE_SHIFT;
         gen8_gtt_pte_t scratch_pte, __iomem *gtt_base =
                 (gen8_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
@@ -995,11 +1525,13 @@ static void gen8_ggtt_clear_range(struct i915_address_space *vm,
 }
 
 static void gen6_ggtt_clear_range(struct i915_address_space *vm,
-                                  unsigned int first_entry,
-                                  unsigned int num_entries,
+                                  uint64_t start,
+                                  uint64_t length,
                                   bool use_scratch)
 {
         struct drm_i915_private *dev_priv = vm->dev->dev_private;
+        unsigned first_entry = start >> PAGE_SHIFT;
+        unsigned num_entries = length >> PAGE_SHIFT;
         gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
                 (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
         const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
@@ -1017,53 +1549,103 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
         readl(gtt_base);
 }
 
-static void i915_ggtt_insert_entries(struct i915_address_space *vm,
-                                     struct sg_table *st,
-                                     unsigned int pg_start,
-                                     enum i915_cache_level cache_level)
+
+static void i915_ggtt_bind_vma(struct i915_vma *vma,
+                               enum i915_cache_level cache_level,
+                               u32 unused)
 {
+        const unsigned long entry = vma->node.start >> PAGE_SHIFT;
         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
 
-        intel_gtt_insert_sg_entries(st, pg_start, flags);
-
+        BUG_ON(!i915_is_ggtt(vma->vm));
+        intel_gtt_insert_sg_entries(vma->obj->pages, entry, flags);
+        vma->obj->has_global_gtt_mapping = 1;
 }
 
 static void i915_ggtt_clear_range(struct i915_address_space *vm,
-                                  unsigned int first_entry,
-                                  unsigned int num_entries,
+                                  uint64_t start,
+                                  uint64_t length,
                                   bool unused)
 {
+        unsigned first_entry = start >> PAGE_SHIFT;
+        unsigned num_entries = length >> PAGE_SHIFT;
         intel_gtt_clear_range(first_entry, num_entries);
 }
 
+static void i915_ggtt_unbind_vma(struct i915_vma *vma)
+{
+        const unsigned int first = vma->node.start >> PAGE_SHIFT;
+        const unsigned int size = vma->obj->base.size >> PAGE_SHIFT;
+
+        BUG_ON(!i915_is_ggtt(vma->vm));
+        vma->obj->has_global_gtt_mapping = 0;
+        intel_gtt_clear_range(first, size);
+}
 
-void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
-                              enum i915_cache_level cache_level)
+static void ggtt_bind_vma(struct i915_vma *vma,
+                          enum i915_cache_level cache_level,
+                          u32 flags)
 {
-        struct drm_device *dev = obj->base.dev;
+        struct drm_device *dev = vma->vm->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
+        struct drm_i915_gem_object *obj = vma->obj;
 
-        dev_priv->gtt.base.insert_entries(&dev_priv->gtt.base, obj->pages,
-                                          entry,
-                                          cache_level);
+        /* If there is no aliasing PPGTT, or the caller needs a global mapping,
+         * or we have a global mapping already but the cacheability flags have
+         * changed, set the global PTEs.
+         *
+         * If there is an aliasing PPGTT it is anecdotally faster, so use that
+         * instead if none of the above hold true.
+         *
+         * NB: A global mapping should only be needed for special regions like
+         * "gtt mappable", SNB errata, or if specified via special execbuf
+         * flags. At all other times, the GPU will use the aliasing PPGTT.
+         */
+        if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) {
+                if (!obj->has_global_gtt_mapping ||
+                    (cache_level != obj->cache_level)) {
+                        vma->vm->insert_entries(vma->vm, obj->pages,
+                                                vma->node.start,
+                                                cache_level);
+                        obj->has_global_gtt_mapping = 1;
+                }
+        }
 
-        obj->has_global_gtt_mapping = 1;
+        if (dev_priv->mm.aliasing_ppgtt &&
+            (!obj->has_aliasing_ppgtt_mapping ||
+             (cache_level != obj->cache_level))) {
+                struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
+                appgtt->base.insert_entries(&appgtt->base,
+                                            vma->obj->pages,
+                                            vma->node.start,
+                                            cache_level);
+                vma->obj->has_aliasing_ppgtt_mapping = 1;
+        }
 }
 
-void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
+static void ggtt_unbind_vma(struct i915_vma *vma)
 {
-        struct drm_device *dev = obj->base.dev;
+        struct drm_device *dev = vma->vm->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
-
-        dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
-                                       entry,
-                                       obj->base.size >> PAGE_SHIFT,
-                                       true);
+        struct drm_i915_gem_object *obj = vma->obj;
+
+        if (obj->has_global_gtt_mapping) {
+                vma->vm->clear_range(vma->vm,
+                                     vma->node.start,
+                                     obj->base.size,
+                                     true);
+                obj->has_global_gtt_mapping = 0;
+        }
 
-        obj->has_global_gtt_mapping = 0;
+        if (obj->has_aliasing_ppgtt_mapping) {
+                struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
+                appgtt->base.clear_range(&appgtt->base,
+                                         vma->node.start,
+                                         obj->base.size,
+                                         true);
+                obj->has_aliasing_ppgtt_mapping = 0;
+        }
 }
 
 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
@@ -1145,29 +1727,14 @@ void i915_gem_setup_global_gtt(struct drm_device *dev,
 
         /* Clear any non-preallocated blocks */
         drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
-                const unsigned long count = (hole_end - hole_start) / PAGE_SIZE;
                 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
                               hole_start, hole_end);
-                ggtt_vm->clear_range(ggtt_vm, hole_start / PAGE_SIZE, count, true);
+                ggtt_vm->clear_range(ggtt_vm, hole_start,
+                                     hole_end - hole_start, true);
         }
 
         /* And finally clear the reserved guard page */
-        ggtt_vm->clear_range(ggtt_vm, end / PAGE_SIZE - 1, 1, true);
-}
-
-static bool
-intel_enable_ppgtt(struct drm_device *dev)
-{
-        if (i915_enable_ppgtt >= 0)
-                return i915_enable_ppgtt;
-
-#ifdef CONFIG_INTEL_IOMMU
-        /* Disable ppgtt on SNB if VT-d is on. */
-        if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
-                return false;
-#endif
-
-        return true;
+        ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
 }
 
 void i915_gem_init_global_gtt(struct drm_device *dev)
@@ -1178,26 +1745,6 @@ void i915_gem_init_global_gtt(struct drm_device *dev)
         gtt_size = dev_priv->gtt.base.total;
         mappable_size = dev_priv->gtt.mappable_end;
 
-        if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
-                int ret;
-
-                if (INTEL_INFO(dev)->gen <= 7) {
-                        /* PPGTT pdes are stolen from global gtt ptes, so shrink the
-                         * aperture accordingly when using aliasing ppgtt. */
-                        gtt_size -= GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
-                }
-
-                i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
-
-                ret = i915_gem_init_aliasing_ppgtt(dev);
-                if (!ret)
-                        return;
-
-                DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
-                drm_mm_takedown(&dev_priv->gtt.base.mm);
-                if (INTEL_INFO(dev)->gen < 8)
-                        gtt_size += GEN6_PPGTT_PD_ENTRIES*PAGE_SIZE;
-        }
         i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
 }
 
@@ -1252,11 +1799,6 @@ static inline unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
         bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
         if (bdw_gmch_ctl)
                 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
-        if (bdw_gmch_ctl > 4) {
-                WARN_ON(!i915_preliminary_hw_support);
-                return 4<<20;
-        }
-
         return bdw_gmch_ctl << 20;
 }
 
@@ -1438,7 +1980,6 @@ static int i915_gmch_probe(struct drm_device *dev,
 
         dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
         dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
-        dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries;
 
         if (unlikely(dev_priv->gtt.do_idle_maps))
                 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
@@ -1493,3 +2034,62 @@ int i915_gem_gtt_init(struct drm_device *dev)
 
         return 0;
 }
+
+static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
+                                              struct i915_address_space *vm)
+{
+        struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
+        if (vma == NULL)
+                return ERR_PTR(-ENOMEM);
+
+        INIT_LIST_HEAD(&vma->vma_link);
+        INIT_LIST_HEAD(&vma->mm_list);
+        INIT_LIST_HEAD(&vma->exec_list);
+        vma->vm = vm;
+        vma->obj = obj;
+
+        switch (INTEL_INFO(vm->dev)->gen) {
+        case 8:
+        case 7:
+        case 6:
+                if (i915_is_ggtt(vm)) {
+                        vma->unbind_vma = ggtt_unbind_vma;
+                        vma->bind_vma = ggtt_bind_vma;
+                } else {
+                        vma->unbind_vma = ppgtt_unbind_vma;
+                        vma->bind_vma = ppgtt_bind_vma;
+                }
+                break;
+        case 5:
+        case 4:
+        case 3:
+        case 2:
+                BUG_ON(!i915_is_ggtt(vm));
+                vma->unbind_vma = i915_ggtt_unbind_vma;
+                vma->bind_vma = i915_ggtt_bind_vma;
+                break;
+        default:
+                BUG();
+        }
+
+        /* Keep GGTT vmas first to make debug easier */
+        if (i915_is_ggtt(vm))
+                list_add(&vma->vma_link, &obj->vma_list);
+        else
+                list_add_tail(&vma->vma_link, &obj->vma_list);
+
+        return vma;
+}
+
+struct i915_vma *
+i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
+                                  struct i915_address_space *vm)
+{
+        struct i915_vma *vma;
+
+        vma = i915_gem_obj_to_vma(obj, vm);
+        if (!vma)
+                vma = __i915_gem_vma_create(obj, vm);
+
+        return vma;
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 28d24caa49f3..62ef55ba061c 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -215,7 +215,7 @@ int i915_gem_init_stolen(struct drm_device *dev)
         int bios_reserved = 0;
 
 #ifdef CONFIG_INTEL_IOMMU
-        if (intel_iommu_gfx_mapped) {
+        if (intel_iommu_gfx_mapped && INTEL_INFO(dev)->gen < 8) {
                 DRM_INFO("DMAR active, disabling use of stolen memory\n");
                 return 0;
         }
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index b13905348048..cb150e8b4336 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -87,7 +87,7 @@
 void
 i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
         uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
 
@@ -294,7 +294,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
                    struct drm_file *file)
 {
         struct drm_i915_gem_set_tiling *args = data;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_object *obj;
         int ret = 0;
 
@@ -308,7 +308,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
                 return -EINVAL;
         }
 
-        if (obj->pin_count || obj->framebuffer_references) {
+        if (i915_gem_obj_is_pinned(obj) || obj->framebuffer_references) {
                 drm_gem_object_unreference_unlocked(&obj->base);
                 return -EBUSY;
         }
@@ -415,7 +415,7 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
                    struct drm_file *file)
 {
         struct drm_i915_gem_get_tiling *args = data;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_object *obj;
 
         obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 990cf8f43efd..12f1d43b2d68 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -238,50 +238,61 @@ static const char *hangcheck_action_to_str(enum intel_ring_hangcheck_action a)
 
 static void i915_ring_error_state(struct drm_i915_error_state_buf *m,
                                   struct drm_device *dev,
-                                  struct drm_i915_error_state *error,
-                                  unsigned ring)
+                                  struct drm_i915_error_ring *ring)
 {
-        BUG_ON(ring >= I915_NUM_RINGS); /* shut up confused gcc */
-        if (!error->ring[ring].valid)
+        if (!ring->valid)
                 return;
 
-        err_printf(m, "%s command stream:\n", ring_str(ring));
-        err_printf(m, "  HEAD: 0x%08x\n", error->head[ring]);
-        err_printf(m, "  TAIL: 0x%08x\n", error->tail[ring]);
-        err_printf(m, "  CTL: 0x%08x\n", error->ctl[ring]);
-        err_printf(m, "  ACTHD: 0x%08x\n", error->acthd[ring]);
-        err_printf(m, "  IPEIR: 0x%08x\n", error->ipeir[ring]);
-        err_printf(m, "  IPEHR: 0x%08x\n", error->ipehr[ring]);
-        err_printf(m, "  INSTDONE: 0x%08x\n", error->instdone[ring]);
+        err_printf(m, "  HEAD: 0x%08x\n", ring->head);
+        err_printf(m, "  TAIL: 0x%08x\n", ring->tail);
+        err_printf(m, "  CTL: 0x%08x\n", ring->ctl);
+        err_printf(m, "  HWS: 0x%08x\n", ring->hws);
+        err_printf(m, "  ACTHD: 0x%08x %08x\n", (u32)(ring->acthd>>32), (u32)ring->acthd);
+        err_printf(m, "  IPEIR: 0x%08x\n", ring->ipeir);
+        err_printf(m, "  IPEHR: 0x%08x\n", ring->ipehr);
+        err_printf(m, "  INSTDONE: 0x%08x\n", ring->instdone);
         if (INTEL_INFO(dev)->gen >= 4) {
-                err_printf(m, "  BBADDR: 0x%08llx\n", error->bbaddr[ring]);
-                err_printf(m, "  BB_STATE: 0x%08x\n", error->bbstate[ring]);
-                err_printf(m, "  INSTPS: 0x%08x\n", error->instps[ring]);
+                err_printf(m, "  BBADDR: 0x%08x %08x\n", (u32)(ring->bbaddr>>32), (u32)ring->bbaddr);
+                err_printf(m, "  BB_STATE: 0x%08x\n", ring->bbstate);
+                err_printf(m, "  INSTPS: 0x%08x\n", ring->instps);
         }
-        err_printf(m, "  INSTPM: 0x%08x\n", error->instpm[ring]);
-        err_printf(m, "  FADDR: 0x%08x\n", error->faddr[ring]);
+        err_printf(m, "  INSTPM: 0x%08x\n", ring->instpm);
+        err_printf(m, "  FADDR: 0x%08x\n", ring->faddr);
         if (INTEL_INFO(dev)->gen >= 6) {
-                err_printf(m, "  RC PSMI: 0x%08x\n", error->rc_psmi[ring]);
-                err_printf(m, "  FAULT_REG: 0x%08x\n", error->fault_reg[ring]);
+                err_printf(m, "  RC PSMI: 0x%08x\n", ring->rc_psmi);
+                err_printf(m, "  FAULT_REG: 0x%08x\n", ring->fault_reg);
                 err_printf(m, "  SYNC_0: 0x%08x [last synced 0x%08x]\n",
-                           error->semaphore_mboxes[ring][0],
-                           error->semaphore_seqno[ring][0]);
+                           ring->semaphore_mboxes[0],
+                           ring->semaphore_seqno[0]);
                 err_printf(m, "  SYNC_1: 0x%08x [last synced 0x%08x]\n",
-                           error->semaphore_mboxes[ring][1],
-                           error->semaphore_seqno[ring][1]);
+                           ring->semaphore_mboxes[1],
+                           ring->semaphore_seqno[1]);
                 if (HAS_VEBOX(dev)) {
                         err_printf(m, "  SYNC_2: 0x%08x [last synced 0x%08x]\n",
-                                   error->semaphore_mboxes[ring][2],
-                                   error->semaphore_seqno[ring][2]);
+                                   ring->semaphore_mboxes[2],
+                                   ring->semaphore_seqno[2]);
                 }
         }
-        err_printf(m, "  seqno: 0x%08x\n", error->seqno[ring]);
-        err_printf(m, "  waiting: %s\n", yesno(error->waiting[ring]));
-        err_printf(m, "  ring->head: 0x%08x\n", error->cpu_ring_head[ring]);
-        err_printf(m, "  ring->tail: 0x%08x\n", error->cpu_ring_tail[ring]);
+        if (USES_PPGTT(dev)) {
+                err_printf(m, "  GFX_MODE: 0x%08x\n", ring->vm_info.gfx_mode);
+
+                if (INTEL_INFO(dev)->gen >= 8) {
+                        int i;
+                        for (i = 0; i < 4; i++)
+                                err_printf(m, "  PDP%d: 0x%016llx\n",
+                                           i, ring->vm_info.pdp[i]);
+                } else {
+                        err_printf(m, "  PP_DIR_BASE: 0x%08x\n",
+                                   ring->vm_info.pp_dir_base);
+                }
+        }
+        err_printf(m, "  seqno: 0x%08x\n", ring->seqno);
+        err_printf(m, "  waiting: %s\n", yesno(ring->waiting));
+        err_printf(m, "  ring->head: 0x%08x\n", ring->cpu_ring_head);
+        err_printf(m, "  ring->tail: 0x%08x\n", ring->cpu_ring_tail);
         err_printf(m, "  hangcheck: %s [%d]\n",
-                   hangcheck_action_to_str(error->hangcheck_action[ring]),
-                   error->hangcheck_score[ring]);
+                   hangcheck_action_to_str(ring->hangcheck_action),
+                   ring->hangcheck_score);
 }
 
 void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...)
@@ -293,22 +304,54 @@ void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...)
         va_end(args);
 }
 
+static void print_error_obj(struct drm_i915_error_state_buf *m,
+                            struct drm_i915_error_object *obj)
+{
+        int page, offset, elt;
+
+        for (page = offset = 0; page < obj->page_count; page++) {
+                for (elt = 0; elt < PAGE_SIZE/4; elt++) {
+                        err_printf(m, "%08x :  %08x\n", offset,
+                                   obj->pages[page][elt]);
+                        offset += 4;
+                }
+        }
+}
+
 int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
                             const struct i915_error_state_file_priv *error_priv)
 {
         struct drm_device *dev = error_priv->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_error_state *error = error_priv->error;
-        int i, j, page, offset, elt;
+        int i, j, offset, elt;
+        int max_hangcheck_score;
 
         if (!error) {
                 err_printf(m, "no error state collected\n");
                 goto out;
         }
 
+        err_printf(m, "%s\n", error->error_msg);
         err_printf(m, "Time: %ld s %ld us\n", error->time.tv_sec,
                    error->time.tv_usec);
         err_printf(m, "Kernel: " UTS_RELEASE "\n");
+        max_hangcheck_score = 0;
+        for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
+                if (error->ring[i].hangcheck_score > max_hangcheck_score)
+                        max_hangcheck_score = error->ring[i].hangcheck_score;
+        }
+        for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
+                if (error->ring[i].hangcheck_score == max_hangcheck_score &&
+                    error->ring[i].pid != -1) {
+                        err_printf(m, "Active process (on ring %s): %s [%d]\n",
+                                   ring_str(i),
+                                   error->ring[i].comm,
+                                   error->ring[i].pid);
+                }
+        }
+        err_printf(m, "Reset count: %u\n", error->reset_count);
+        err_printf(m, "Suspend count: %u\n", error->suspend_count);
         err_printf(m, "PCI ID: 0x%04x\n", dev->pdev->device);
         err_printf(m, "EIR: 0x%08x\n", error->eir);
         err_printf(m, "IER: 0x%08x\n", error->ier);
@@ -333,8 +376,10 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
         if (INTEL_INFO(dev)->gen == 7)
                 err_printf(m, "ERR_INT: 0x%08x\n", error->err_int);
 
-        for (i = 0; i < ARRAY_SIZE(error->ring); i++)
-                i915_ring_error_state(m, dev, error, i);
+        for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
+                err_printf(m, "%s command stream:\n", ring_str(i));
+                i915_ring_error_state(m, dev, &error->ring[i]);
+        }
 
         if (error->active_bo)
                 print_error_buffers(m, "Active",
@@ -349,18 +394,23 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
         for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
                 struct drm_i915_error_object *obj;
 
-                if ((obj = error->ring[i].batchbuffer)) {
-                        err_printf(m, "%s --- gtt_offset = 0x%08x\n",
-                                   dev_priv->ring[i].name,
+                obj = error->ring[i].batchbuffer;
+                if (obj) {
+                        err_puts(m, dev_priv->ring[i].name);
+                        if (error->ring[i].pid != -1)
+                                err_printf(m, " (submitted by %s [%d])",
+                                           error->ring[i].comm,
+                                           error->ring[i].pid);
+                        err_printf(m, " --- gtt_offset = 0x%08x\n",
                                    obj->gtt_offset);
-                        offset = 0;
-                        for (page = 0; page < obj->page_count; page++) {
-                                for (elt = 0; elt < PAGE_SIZE/4; elt++) {
-                                        err_printf(m, "%08x :  %08x\n", offset,
-                                                   obj->pages[page][elt]);
-                                        offset += 4;
-                                }
-                        }
+                        print_error_obj(m, obj);
+                }
+
+                obj = error->ring[i].wa_batchbuffer;
+                if (obj) {
+                        err_printf(m, "%s (w/a) --- gtt_offset = 0x%08x\n",
+                                   dev_priv->ring[i].name, obj->gtt_offset);
+                        print_error_obj(m, obj);
                 }
 
                 if (error->ring[i].num_requests) {
@@ -379,14 +429,22 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
                         err_printf(m, "%s --- ringbuffer = 0x%08x\n",
                                    dev_priv->ring[i].name,
                                    obj->gtt_offset);
+                        print_error_obj(m, obj);
+                }
+
+                if ((obj = error->ring[i].hws_page)) {
+                        err_printf(m, "%s --- HW Status = 0x%08x\n",
+                                   dev_priv->ring[i].name,
+                                   obj->gtt_offset);
                         offset = 0;
-                        for (page = 0; page < obj->page_count; page++) {
-                                for (elt = 0; elt < PAGE_SIZE/4; elt++) {
-                                        err_printf(m, "%08x :  %08x\n",
-                                                   offset,
-                                                   obj->pages[page][elt]);
-                                        offset += 4;
-                                }
+                        for (elt = 0; elt < PAGE_SIZE/16; elt += 4) {
+                                err_printf(m, "[%04x] %08x %08x %08x %08x\n",
+                                           offset,
+                                           obj->pages[0][elt],
+                                           obj->pages[0][elt+1],
+                                           obj->pages[0][elt+2],
+                                           obj->pages[0][elt+3]);
+                                        offset += 16;
                         }
                 }
 
@@ -472,6 +530,7 @@ static void i915_error_state_free(struct kref *error_ref)
         for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
                 i915_error_object_free(error->ring[i].batchbuffer);
                 i915_error_object_free(error->ring[i].ringbuffer);
+                i915_error_object_free(error->ring[i].hws_page);
                 i915_error_object_free(error->ring[i].ctx);
                 kfree(error->ring[i].requests);
         }
@@ -485,6 +544,7 @@ static void i915_error_state_free(struct kref *error_ref)
 static struct drm_i915_error_object *
 i915_error_object_create_sized(struct drm_i915_private *dev_priv,
                                struct drm_i915_gem_object *src,
+                               struct i915_address_space *vm,
                                const int num_pages)
 {
         struct drm_i915_error_object *dst;
@@ -498,7 +558,7 @@ i915_error_object_create_sized(struct drm_i915_private *dev_priv,
         if (dst == NULL)
                 return NULL;
 
-        reloc_offset = dst->gtt_offset = i915_gem_obj_ggtt_offset(src);
+        reloc_offset = dst->gtt_offset = i915_gem_obj_offset(src, vm);
         for (i = 0; i < num_pages; i++) {
                 unsigned long flags;
                 void *d;
@@ -508,8 +568,10 @@ i915_error_object_create_sized(struct drm_i915_private *dev_priv,
                         goto unwind;
 
                 local_irq_save(flags);
-                if (reloc_offset < dev_priv->gtt.mappable_end &&
-                    src->has_global_gtt_mapping) {
+                if (src->cache_level == I915_CACHE_NONE &&
+                    reloc_offset < dev_priv->gtt.mappable_end &&
+                    src->has_global_gtt_mapping &&
+                    i915_is_ggtt(vm)) {
                         void __iomem *s;
 
                         /* Simply ignore tiling or any overlapping fence.
@@ -559,8 +621,12 @@ unwind:
         kfree(dst);
         return NULL;
 }
-#define i915_error_object_create(dev_priv, src) \
-        i915_error_object_create_sized((dev_priv), (src), \
+#define i915_error_object_create(dev_priv, src, vm) \
+        i915_error_object_create_sized((dev_priv), (src), (vm), \
+                                       (src)->base.size>>PAGE_SHIFT)
+
+#define i915_error_ggtt_object_create(dev_priv, src) \
+        i915_error_object_create_sized((dev_priv), (src), &(dev_priv)->gtt.base, \
                                        (src)->base.size>>PAGE_SHIFT)
 
 static void capture_bo(struct drm_i915_error_buffer *err,
@@ -575,7 +641,7 @@ static void capture_bo(struct drm_i915_error_buffer *err,
         err->write_domain = obj->base.write_domain;
         err->fence_reg = obj->fence_reg;
         err->pinned = 0;
-        if (obj->pin_count > 0)
+        if (i915_gem_obj_is_pinned(obj))
                 err->pinned = 1;
         if (obj->user_pin_count > 0)
                 err->pinned = -1;
@@ -608,7 +674,7 @@ static u32 capture_pinned_bo(struct drm_i915_error_buffer *err,
         int i = 0;
 
         list_for_each_entry(obj, head, global_list) {
-                if (obj->pin_count == 0)
+                if (!i915_gem_obj_is_pinned(obj))
                         continue;
 
                 capture_bo(err++, obj);
@@ -619,6 +685,39 @@ static u32 capture_pinned_bo(struct drm_i915_error_buffer *err,
         return i;
 }
 
+/* Generate a semi-unique error code. The code is not meant to have meaning, The
+ * code's only purpose is to try to prevent false duplicated bug reports by
+ * grossly estimating a GPU error state.
+ *
+ * TODO Ideally, hashing the batchbuffer would be a very nice way to determine
+ * the hang if we could strip the GTT offset information from it.
+ *
+ * It's only a small step better than a random number in its current form.
+ */
+static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv,
+                                         struct drm_i915_error_state *error,
+                                         int *ring_id)
+{
+        uint32_t error_code = 0;
+        int i;
+
+        /* IPEHR would be an ideal way to detect errors, as it's the gross
+         * measure of "the command that hung." However, has some very common
+         * synchronization commands which almost always appear in the case
+         * strictly a client bug. Use instdone to differentiate those some.
+         */
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                if (error->ring[i].hangcheck_action == HANGCHECK_HUNG) {
+                        if (ring_id)
+                                *ring_id = i;
+
+                        return error->ring[i].ipehr ^ error->ring[i].instdone;
+                }
+        }
+
+        return error_code;
+}
+
 static void i915_gem_record_fences(struct drm_device *dev,
                                    struct drm_i915_error_state *error)
 {
@@ -652,107 +751,114 @@ static void i915_gem_record_fences(struct drm_device *dev,
         }
 }
 
-static struct drm_i915_error_object *
-i915_error_first_batchbuffer(struct drm_i915_private *dev_priv,
-                             struct intel_ring_buffer *ring)
-{
-        struct i915_address_space *vm;
-        struct i915_vma *vma;
-        struct drm_i915_gem_object *obj;
-        u32 seqno;
-
-        if (!ring->get_seqno)
-                return NULL;
-
-        if (HAS_BROKEN_CS_TLB(dev_priv->dev)) {
-                u32 acthd = I915_READ(ACTHD);
-
-                if (WARN_ON(ring->id != RCS))
-                        return NULL;
-
-                obj = ring->scratch.obj;
-                if (obj != NULL &&
-                    acthd >= i915_gem_obj_ggtt_offset(obj) &&
-                    acthd < i915_gem_obj_ggtt_offset(obj) + obj->base.size)
-                        return i915_error_object_create(dev_priv, obj);
-        }
-
-        seqno = ring->get_seqno(ring, false);
-        list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
-                list_for_each_entry(vma, &vm->active_list, mm_list) {
-                        obj = vma->obj;
-                        if (obj->ring != ring)
-                                continue;
-
-                        if (i915_seqno_passed(seqno, obj->last_read_seqno))
-                                continue;
-
-                        if ((obj->base.read_domains & I915_GEM_DOMAIN_COMMAND) == 0)
-                                continue;
-
-                        /* We need to copy these to an anonymous buffer as the simplest
-                         * method to avoid being overwritten by userspace.
-                         */
-                        return i915_error_object_create(dev_priv, obj);
-                }
-        }
-
-        return NULL;
-}
-
 static void i915_record_ring_state(struct drm_device *dev,
-                                   struct drm_i915_error_state *error,
-                                   struct intel_ring_buffer *ring)
+                                   struct intel_ring_buffer *ring,
+                                   struct drm_i915_error_ring *ering)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         if (INTEL_INFO(dev)->gen >= 6) {
-                error->rc_psmi[ring->id] = I915_READ(ring->mmio_base + 0x50);
-                error->fault_reg[ring->id] = I915_READ(RING_FAULT_REG(ring));
-                error->semaphore_mboxes[ring->id][0]
+                ering->rc_psmi = I915_READ(ring->mmio_base + 0x50);
+                ering->fault_reg = I915_READ(RING_FAULT_REG(ring));
+                ering->semaphore_mboxes[0]
                         = I915_READ(RING_SYNC_0(ring->mmio_base));
-                error->semaphore_mboxes[ring->id][1]
+                ering->semaphore_mboxes[1]
                         = I915_READ(RING_SYNC_1(ring->mmio_base));
-                error->semaphore_seqno[ring->id][0] = ring->sync_seqno[0];
-                error->semaphore_seqno[ring->id][1] = ring->sync_seqno[1];
+                ering->semaphore_seqno[0] = ring->sync_seqno[0];
+                ering->semaphore_seqno[1] = ring->sync_seqno[1];
         }
 
         if (HAS_VEBOX(dev)) {
-                error->semaphore_mboxes[ring->id][2] =
+                ering->semaphore_mboxes[2] =
                         I915_READ(RING_SYNC_2(ring->mmio_base));
-                error->semaphore_seqno[ring->id][2] = ring->sync_seqno[2];
+                ering->semaphore_seqno[2] = ring->sync_seqno[2];
         }
 
         if (INTEL_INFO(dev)->gen >= 4) {
-                error->faddr[ring->id] = I915_READ(RING_DMA_FADD(ring->mmio_base));
-                error->ipeir[ring->id] = I915_READ(RING_IPEIR(ring->mmio_base));
-                error->ipehr[ring->id] = I915_READ(RING_IPEHR(ring->mmio_base));
-                error->instdone[ring->id] = I915_READ(RING_INSTDONE(ring->mmio_base));
-                error->instps[ring->id] = I915_READ(RING_INSTPS(ring->mmio_base));
-                error->bbaddr[ring->id] = I915_READ(RING_BBADDR(ring->mmio_base));
+                ering->faddr = I915_READ(RING_DMA_FADD(ring->mmio_base));
+                ering->ipeir = I915_READ(RING_IPEIR(ring->mmio_base));
+                ering->ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
+                ering->instdone = I915_READ(RING_INSTDONE(ring->mmio_base));
+                ering->instps = I915_READ(RING_INSTPS(ring->mmio_base));
+                ering->bbaddr = I915_READ(RING_BBADDR(ring->mmio_base));
                 if (INTEL_INFO(dev)->gen >= 8)
-                        error->bbaddr[ring->id] |= (u64) I915_READ(RING_BBADDR_UDW(ring->mmio_base)) << 32;
-                error->bbstate[ring->id] = I915_READ(RING_BBSTATE(ring->mmio_base));
+                        ering->bbaddr |= (u64) I915_READ(RING_BBADDR_UDW(ring->mmio_base)) << 32;
+                ering->bbstate = I915_READ(RING_BBSTATE(ring->mmio_base));
         } else {
-                error->faddr[ring->id] = I915_READ(DMA_FADD_I8XX);
-                error->ipeir[ring->id] = I915_READ(IPEIR);
-                error->ipehr[ring->id] = I915_READ(IPEHR);
-                error->instdone[ring->id] = I915_READ(INSTDONE);
+                ering->faddr = I915_READ(DMA_FADD_I8XX);
+                ering->ipeir = I915_READ(IPEIR);
+                ering->ipehr = I915_READ(IPEHR);
+                ering->instdone = I915_READ(INSTDONE);
+        }
+
+        ering->waiting = waitqueue_active(&ring->irq_queue);
+        ering->instpm = I915_READ(RING_INSTPM(ring->mmio_base));
+        ering->seqno = ring->get_seqno(ring, false);
+        ering->acthd = intel_ring_get_active_head(ring);
+        ering->head = I915_READ_HEAD(ring);
+        ering->tail = I915_READ_TAIL(ring);
+        ering->ctl = I915_READ_CTL(ring);
+
+        if (I915_NEED_GFX_HWS(dev)) {
+                int mmio;
+
+                if (IS_GEN7(dev)) {
+                        switch (ring->id) {
+                        default:
+                        case RCS:
+                                mmio = RENDER_HWS_PGA_GEN7;
+                                break;
+                        case BCS:
+                                mmio = BLT_HWS_PGA_GEN7;
+                                break;
+                        case VCS:
+                                mmio = BSD_HWS_PGA_GEN7;
+                                break;
+                        case VECS:
+                                mmio = VEBOX_HWS_PGA_GEN7;
+                                break;
+                        }
+                } else if (IS_GEN6(ring->dev)) {
+                        mmio = RING_HWS_PGA_GEN6(ring->mmio_base);
+                } else {
+                        /* XXX: gen8 returns to sanity */
+                        mmio = RING_HWS_PGA(ring->mmio_base);
+                }
+
+                ering->hws = I915_READ(mmio);
         }
 
-        error->waiting[ring->id] = waitqueue_active(&ring->irq_queue);
-        error->instpm[ring->id] = I915_READ(RING_INSTPM(ring->mmio_base));
-        error->seqno[ring->id] = ring->get_seqno(ring, false);
-        error->acthd[ring->id] = intel_ring_get_active_head(ring);
-        error->head[ring->id] = I915_READ_HEAD(ring);
-        error->tail[ring->id] = I915_READ_TAIL(ring);
-        error->ctl[ring->id] = I915_READ_CTL(ring);
+        ering->cpu_ring_head = ring->head;
+        ering->cpu_ring_tail = ring->tail;
+
+        ering->hangcheck_score = ring->hangcheck.score;
+        ering->hangcheck_action = ring->hangcheck.action;
+
+        if (USES_PPGTT(dev)) {
+                int i;
 
-        error->cpu_ring_head[ring->id] = ring->head;
-        error->cpu_ring_tail[ring->id] = ring->tail;
+                ering->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(ring));
 
-        error->hangcheck_score[ring->id] = ring->hangcheck.score;
-        error->hangcheck_action[ring->id] = ring->hangcheck.action;
+                switch (INTEL_INFO(dev)->gen) {
+                case 8:
+                        for (i = 0; i < 4; i++) {
+                                ering->vm_info.pdp[i] =
+                                        I915_READ(GEN8_RING_PDP_UDW(ring, i));
+                                ering->vm_info.pdp[i] <<= 32;
+                                ering->vm_info.pdp[i] |=
+                                        I915_READ(GEN8_RING_PDP_LDW(ring, i));
+                        }
+                        break;
+                case 7:
+                        ering->vm_info.pp_dir_base =
+                                I915_READ(RING_PP_DIR_BASE(ring));
+                        break;
+                case 6:
+                        ering->vm_info.pp_dir_base =
+                                I915_READ(RING_PP_DIR_BASE_READ(ring));
+                        break;
+                }
+        }
 }
 
 
@@ -770,7 +876,9 @@ static void i915_gem_record_active_context(struct intel_ring_buffer *ring,
         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
                 if ((error->ccid & PAGE_MASK) == i915_gem_obj_ggtt_offset(obj)) {
                         ering->ctx = i915_error_object_create_sized(dev_priv,
-                                                                    obj, 1);
+                                                                    obj,
+                                                                    &dev_priv->gtt.base,
+                                                                    1);
                         break;
                 }
         }
@@ -791,14 +899,48 @@ static void i915_gem_record_rings(struct drm_device *dev,
 
                 error->ring[i].valid = true;
 
-                i915_record_ring_state(dev, error, ring);
+                i915_record_ring_state(dev, ring, &error->ring[i]);
 
-                error->ring[i].batchbuffer =
-                        i915_error_first_batchbuffer(dev_priv, ring);
+                error->ring[i].pid = -1;
+                request = i915_gem_find_active_request(ring);
+                if (request) {
+                        /* We need to copy these to an anonymous buffer
+                         * as the simplest method to avoid being overwritten
+                         * by userspace.
+                         */
+                        error->ring[i].batchbuffer =
+                                i915_error_object_create(dev_priv,
+                                                         request->batch_obj,
+                                                         request->ctx ?
+                                                         request->ctx->vm :
+                                                         &dev_priv->gtt.base);
+
+                        if (HAS_BROKEN_CS_TLB(dev_priv->dev) &&
+                            ring->scratch.obj)
+                                error->ring[i].wa_batchbuffer =
+                                        i915_error_ggtt_object_create(dev_priv,
+                                                             ring->scratch.obj);
+
+                        if (request->file_priv) {
+                                struct task_struct *task;
+
+                                rcu_read_lock();
+                                task = pid_task(request->file_priv->file->pid,
+                                                PIDTYPE_PID);
+                                if (task) {
+                                        strcpy(error->ring[i].comm, task->comm);
+                                        error->ring[i].pid = task->pid;
+                                }
+                                rcu_read_unlock();
+                        }
+                }
 
                 error->ring[i].ringbuffer =
-                        i915_error_object_create(dev_priv, ring->obj);
+                        i915_error_ggtt_object_create(dev_priv, ring->obj);
 
+                if (ring->status_page.obj)
+                        error->ring[i].hws_page =
+                                i915_error_ggtt_object_create(dev_priv, ring->status_page.obj);
 
                 i915_gem_record_active_context(ring, error, &error->ring[i]);
 
@@ -845,7 +987,7 @@ static void i915_gem_capture_vm(struct drm_i915_private *dev_priv,
                 i++;
         error->active_bo_count[ndx] = i;
         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
-                if (obj->pin_count)
+                if (i915_gem_obj_is_pinned(obj))
                         i++;
         error->pinned_bo_count[ndx] = i - error->active_bo_count[ndx];
 
@@ -879,11 +1021,6 @@ static void i915_gem_capture_buffers(struct drm_i915_private *dev_priv,
         list_for_each_entry(vm, &dev_priv->vm_list, global_link)
                 cnt++;
 
-        if (WARN(cnt > 1, "Multiple VMs not yet supported\n"))
-                cnt = 1;
-
-        vm = &dev_priv->gtt.base;
-
         error->active_bo = kcalloc(cnt, sizeof(*error->active_bo), GFP_ATOMIC);
         error->pinned_bo = kcalloc(cnt, sizeof(*error->pinned_bo), GFP_ATOMIC);
         error->active_bo_count = kcalloc(cnt, sizeof(*error->active_bo_count),
@@ -895,6 +1032,108 @@ static void i915_gem_capture_buffers(struct drm_i915_private *dev_priv,
                 i915_gem_capture_vm(dev_priv, error, vm, i++);
 }
 
+/* Capture all registers which don't fit into another category. */
+static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
+                                   struct drm_i915_error_state *error)
+{
+        struct drm_device *dev = dev_priv->dev;
+        int pipe;
+
+        /* General organization
+         * 1. Registers specific to a single generation
+         * 2. Registers which belong to multiple generations
+         * 3. Feature specific registers.
+         * 4. Everything else
+         * Please try to follow the order.
+         */
+
+        /* 1: Registers specific to a single generation */
+        if (IS_VALLEYVIEW(dev)) {
+                error->ier = I915_READ(GTIER) | I915_READ(VLV_IER);
+                error->forcewake = I915_READ(FORCEWAKE_VLV);
+        }
+
+        if (IS_GEN7(dev))
+                error->err_int = I915_READ(GEN7_ERR_INT);
+
+        if (IS_GEN6(dev)) {
+                error->forcewake = I915_READ(FORCEWAKE);
+                error->gab_ctl = I915_READ(GAB_CTL);
+                error->gfx_mode = I915_READ(GFX_MODE);
+        }
+
+        if (IS_GEN2(dev))
+                error->ier = I915_READ16(IER);
+
+        /* 2: Registers which belong to multiple generations */
+        if (INTEL_INFO(dev)->gen >= 7)
+                error->forcewake = I915_READ(FORCEWAKE_MT);
+
+        if (INTEL_INFO(dev)->gen >= 6) {
+                error->derrmr = I915_READ(DERRMR);
+                error->error = I915_READ(ERROR_GEN6);
+                error->done_reg = I915_READ(DONE_REG);
+        }
+
+        /* 3: Feature specific registers */
+        if (IS_GEN6(dev) || IS_GEN7(dev)) {
+                error->gam_ecochk = I915_READ(GAM_ECOCHK);
+                error->gac_eco = I915_READ(GAC_ECO_BITS);
+        }
+
+        /* 4: Everything else */
+        if (HAS_HW_CONTEXTS(dev))
+                error->ccid = I915_READ(CCID);
+
+        if (HAS_PCH_SPLIT(dev))
+                error->ier = I915_READ(DEIER) | I915_READ(GTIER);
+        else {
+                error->ier = I915_READ(IER);
+                for_each_pipe(pipe)
+                        error->pipestat[pipe] = I915_READ(PIPESTAT(pipe));
+        }
+
+        /* 4: Everything else */
+        error->eir = I915_READ(EIR);
+        error->pgtbl_er = I915_READ(PGTBL_ER);
+
+        i915_get_extra_instdone(dev, error->extra_instdone);
+}
+
+static void i915_error_capture_msg(struct drm_device *dev,
+                                   struct drm_i915_error_state *error,
+                                   bool wedged,
+                                   const char *error_msg)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 ecode;
+        int ring_id = -1, len;
+
+        ecode = i915_error_generate_code(dev_priv, error, &ring_id);
+
+        len = scnprintf(error->error_msg, sizeof(error->error_msg),
+                        "GPU HANG: ecode %d:0x%08x", ring_id, ecode);
+
+        if (ring_id != -1 && error->ring[ring_id].pid != -1)
+                len += scnprintf(error->error_msg + len,
+                                 sizeof(error->error_msg) - len,
+                                 ", in %s [%d]",
+                                 error->ring[ring_id].comm,
+                                 error->ring[ring_id].pid);
+
+        scnprintf(error->error_msg + len, sizeof(error->error_msg) - len,
+                  ", reason: %s, action: %s",
+                  error_msg,
+                  wedged ? "reset" : "continue");
+}
+
+static void i915_capture_gen_state(struct drm_i915_private *dev_priv,
+                                   struct drm_i915_error_state *error)
+{
+        error->reset_count = i915_reset_count(&dev_priv->gpu_error);
+        error->suspend_count = dev_priv->suspend_count;
+}
+
 /**
  * i915_capture_error_state - capture an error record for later analysis
  * @dev: drm device
@@ -904,18 +1143,13 @@ static void i915_gem_capture_buffers(struct drm_i915_private *dev_priv,
  * out a structure which becomes available in debugfs for user level tools
  * to pick up.
  */
-void i915_capture_error_state(struct drm_device *dev)
+void i915_capture_error_state(struct drm_device *dev, bool wedged,
+                              const char *error_msg)
 {
+        static bool warned;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_error_state *error;
         unsigned long flags;
-        int pipe;
-
-        spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
-        error = dev_priv->gpu_error.first_error;
-        spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
-        if (error)
-                return;
 
         /* Account for pipe specific data like PIPE*STAT */
         error = kzalloc(sizeof(*error), GFP_ATOMIC);
@@ -924,52 +1158,10 @@ void i915_capture_error_state(struct drm_device *dev)
                 return;
         }
 
-        DRM_INFO("GPU crash dump saved to /sys/class/drm/card%d/error\n",
-                 dev->primary->index);
-        DRM_INFO("GPU hangs can indicate a bug anywhere in the entire gfx stack, including userspace.\n");
-        DRM_INFO("Please file a _new_ bug report on bugs.freedesktop.org against DRI -> DRM/Intel\n");
-        DRM_INFO("drm/i915 developers can then reassign to the right component if it's not a kernel issue.\n");
-        DRM_INFO("The gpu crash dump is required to analyze gpu hangs, so please always attach it.\n");
-
         kref_init(&error->ref);
-        error->eir = I915_READ(EIR);
-        error->pgtbl_er = I915_READ(PGTBL_ER);
-        if (HAS_HW_CONTEXTS(dev))
-                error->ccid = I915_READ(CCID);
-
-        if (HAS_PCH_SPLIT(dev))
-                error->ier = I915_READ(DEIER) | I915_READ(GTIER);
-        else if (IS_VALLEYVIEW(dev))
-                error->ier = I915_READ(GTIER) | I915_READ(VLV_IER);
-        else if (IS_GEN2(dev))
-                error->ier = I915_READ16(IER);
-        else
-                error->ier = I915_READ(IER);
-
-        if (INTEL_INFO(dev)->gen >= 6)
-                error->derrmr = I915_READ(DERRMR);
-
-        if (IS_VALLEYVIEW(dev))
-                error->forcewake = I915_READ(FORCEWAKE_VLV);
-        else if (INTEL_INFO(dev)->gen >= 7)
-                error->forcewake = I915_READ(FORCEWAKE_MT);
-        else if (INTEL_INFO(dev)->gen == 6)
-                error->forcewake = I915_READ(FORCEWAKE);
-
-        if (!HAS_PCH_SPLIT(dev))
-                for_each_pipe(pipe)
-                        error->pipestat[pipe] = I915_READ(PIPESTAT(pipe));
-
-        if (INTEL_INFO(dev)->gen >= 6) {
-                error->error = I915_READ(ERROR_GEN6);
-                error->done_reg = I915_READ(DONE_REG);
-        }
-
-        if (INTEL_INFO(dev)->gen == 7)
-                error->err_int = I915_READ(GEN7_ERR_INT);
-
-        i915_get_extra_instdone(dev, error->extra_instdone);
 
+        i915_capture_gen_state(dev_priv, error);
+        i915_capture_reg_state(dev_priv, error);
         i915_gem_capture_buffers(dev_priv, error);
         i915_gem_record_fences(dev, error);
         i915_gem_record_rings(dev, error);
@@ -979,6 +1171,9 @@ void i915_capture_error_state(struct drm_device *dev)
         error->overlay = intel_overlay_capture_error_state(dev);
         error->display = intel_display_capture_error_state(dev);
 
+        i915_error_capture_msg(dev, error, wedged, error_msg);
+        DRM_INFO("%s\n", error->error_msg);
+
         spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
         if (dev_priv->gpu_error.first_error == NULL) {
                 dev_priv->gpu_error.first_error = error;
@@ -986,8 +1181,19 @@ void i915_capture_error_state(struct drm_device *dev)
         }
         spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
 
-        if (error)
+        if (error) {
                 i915_error_state_free(&error->ref);
+                return;
+        }
+
+        if (!warned) {
+                DRM_INFO("GPU hangs can indicate a bug anywhere in the entire gfx stack, including userspace.\n");
+                DRM_INFO("Please file a _new_ bug report on bugs.freedesktop.org against DRI -> DRM/Intel\n");
+                DRM_INFO("drm/i915 developers can then reassign to the right component if it's not a kernel issue.\n");
+                DRM_INFO("The gpu crash dump is required to analyze gpu hangs, so please always attach it.\n");
+                DRM_INFO("GPU crash dump saved to /sys/class/drm/card%d/error\n", dev->primary->index);
+                warned = true;
+        }
 }
 
 void i915_error_state_get(struct drm_device *dev,
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index d554169ac592..7753249b3a95 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -82,13 +82,13 @@ static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
 
 /* For display hotplug interrupt */
 static void
-ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
+ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
 {
         assert_spin_locked(&dev_priv->irq_lock);
 
-        if (dev_priv->pc8.irqs_disabled) {
+        if (dev_priv->pm.irqs_disabled) {
                 WARN(1, "IRQs disabled\n");
-                dev_priv->pc8.regsave.deimr &= ~mask;
+                dev_priv->pm.regsave.deimr &= ~mask;
                 return;
         }
 
@@ -100,13 +100,13 @@ ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
 }
 
 static void
-ironlake_disable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
+ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
 {
         assert_spin_locked(&dev_priv->irq_lock);
 
-        if (dev_priv->pc8.irqs_disabled) {
+        if (dev_priv->pm.irqs_disabled) {
                 WARN(1, "IRQs disabled\n");
-                dev_priv->pc8.regsave.deimr |= mask;
+                dev_priv->pm.regsave.deimr |= mask;
                 return;
         }
 
@@ -129,10 +129,10 @@ static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
 {
         assert_spin_locked(&dev_priv->irq_lock);
 
-        if (dev_priv->pc8.irqs_disabled) {
+        if (dev_priv->pm.irqs_disabled) {
                 WARN(1, "IRQs disabled\n");
-                dev_priv->pc8.regsave.gtimr &= ~interrupt_mask;
-                dev_priv->pc8.regsave.gtimr |= (~enabled_irq_mask &
+                dev_priv->pm.regsave.gtimr &= ~interrupt_mask;
+                dev_priv->pm.regsave.gtimr |= (~enabled_irq_mask &
                                                 interrupt_mask);
                 return;
         }
@@ -167,10 +167,10 @@ static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
 
         assert_spin_locked(&dev_priv->irq_lock);
 
-        if (dev_priv->pc8.irqs_disabled) {
+        if (dev_priv->pm.irqs_disabled) {
                 WARN(1, "IRQs disabled\n");
-                dev_priv->pc8.regsave.gen6_pmimr &= ~interrupt_mask;
-                dev_priv->pc8.regsave.gen6_pmimr |= (~enabled_irq_mask &
+                dev_priv->pm.regsave.gen6_pmimr &= ~interrupt_mask;
+                dev_priv->pm.regsave.gen6_pmimr |= (~enabled_irq_mask &
                                                      interrupt_mask);
                 return;
         }
@@ -232,6 +232,18 @@ static bool cpt_can_enable_serr_int(struct drm_device *dev)
         return true;
 }
 
+static void i9xx_clear_fifo_underrun(struct drm_device *dev, enum pipe pipe)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 reg = PIPESTAT(pipe);
+        u32 pipestat = I915_READ(reg) & 0x7fff0000;
+
+        assert_spin_locked(&dev_priv->irq_lock);
+
+        I915_WRITE(reg, pipestat | PIPE_FIFO_UNDERRUN_STATUS);
+        POSTING_READ(reg);
+}
+
 static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev,
                                                  enum pipe pipe, bool enable)
 {
@@ -301,11 +313,11 @@ static void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
 
         assert_spin_locked(&dev_priv->irq_lock);
 
-        if (dev_priv->pc8.irqs_disabled &&
+        if (dev_priv->pm.irqs_disabled &&
             (interrupt_mask & SDE_HOTPLUG_MASK_CPT)) {
                 WARN(1, "IRQs disabled\n");
-                dev_priv->pc8.regsave.sdeimr &= ~interrupt_mask;
-                dev_priv->pc8.regsave.sdeimr |= (~enabled_irq_mask &
+                dev_priv->pm.regsave.sdeimr &= ~interrupt_mask;
+                dev_priv->pm.regsave.sdeimr |= (~enabled_irq_mask &
                                                  interrupt_mask);
                 return;
         }
@@ -375,16 +387,15 @@ static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
  *
  * Returns the previous state of underrun reporting.
  */
-bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
-                                           enum pipe pipe, bool enable)
+bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
+                                             enum pipe pipe, bool enable)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        unsigned long flags;
         bool ret;
 
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
+        assert_spin_locked(&dev_priv->irq_lock);
 
         ret = !intel_crtc->cpu_fifo_underrun_disabled;
 
@@ -393,7 +404,9 @@ bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
 
         intel_crtc->cpu_fifo_underrun_disabled = !enable;
 
-        if (IS_GEN5(dev) || IS_GEN6(dev))
+        if (enable && (INTEL_INFO(dev)->gen < 5 || IS_VALLEYVIEW(dev)))
+                i9xx_clear_fifo_underrun(dev, pipe);
+        else if (IS_GEN5(dev) || IS_GEN6(dev))
                 ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
         else if (IS_GEN7(dev))
                 ivybridge_set_fifo_underrun_reporting(dev, pipe, enable);
@@ -401,10 +414,33 @@ bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
                 broadwell_set_fifo_underrun_reporting(dev, pipe, enable);
 
 done:
+        return ret;
+}
+
+bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
+                                           enum pipe pipe, bool enable)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long flags;
+        bool ret;
+
+        spin_lock_irqsave(&dev_priv->irq_lock, flags);
+        ret = __intel_set_cpu_fifo_underrun_reporting(dev, pipe, enable);
         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+
         return ret;
 }
 
+static bool __cpu_fifo_underrun_reporting_enabled(struct drm_device *dev,
+                                                  enum pipe pipe)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+
+        return !intel_crtc->cpu_fifo_underrun_disabled;
+}
+
 /**
  * intel_set_pch_fifo_underrun_reporting - enable/disable FIFO underrun messages
  * @dev: drm device
@@ -458,45 +494,109 @@ done:
 }
 
 
-void
-i915_enable_pipestat(drm_i915_private_t *dev_priv, enum pipe pipe, u32 mask)
+static void
+__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
+                       u32 enable_mask, u32 status_mask)
 {
         u32 reg = PIPESTAT(pipe);
-        u32 pipestat = I915_READ(reg) & 0x7fff0000;
+        u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
 
         assert_spin_locked(&dev_priv->irq_lock);
 
-        if ((pipestat & mask) == mask)
+        if (WARN_ON_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
+                         status_mask & ~PIPESTAT_INT_STATUS_MASK))
                 return;
 
+        if ((pipestat & enable_mask) == enable_mask)
+                return;
+
+        dev_priv->pipestat_irq_mask[pipe] |= status_mask;
+
         /* Enable the interrupt, clear any pending status */
-        pipestat |= mask | (mask >> 16);
+        pipestat |= enable_mask | status_mask;
         I915_WRITE(reg, pipestat);
         POSTING_READ(reg);
 }
 
-void
-i915_disable_pipestat(drm_i915_private_t *dev_priv, enum pipe pipe, u32 mask)
+static void
+__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
+                        u32 enable_mask, u32 status_mask)
 {
         u32 reg = PIPESTAT(pipe);
-        u32 pipestat = I915_READ(reg) & 0x7fff0000;
+        u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
 
         assert_spin_locked(&dev_priv->irq_lock);
 
-        if ((pipestat & mask) == 0)
+        if (WARN_ON_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
+                         status_mask & ~PIPESTAT_INT_STATUS_MASK))
+                return;
+
+        if ((pipestat & enable_mask) == 0)
                 return;
 
-        pipestat &= ~mask;
+        dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
+
+        pipestat &= ~enable_mask;
         I915_WRITE(reg, pipestat);
         POSTING_READ(reg);
 }
 
+static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
+{
+        u32 enable_mask = status_mask << 16;
+
+        /*
+         * On pipe A we don't support the PSR interrupt yet, on pipe B the
+         * same bit MBZ.
+         */
+        if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
+                return 0;
+
+        enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
+                         SPRITE0_FLIP_DONE_INT_EN_VLV |
+                         SPRITE1_FLIP_DONE_INT_EN_VLV);
+        if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
+                enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
+        if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
+                enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
+
+        return enable_mask;
+}
+
+void
+i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
+                     u32 status_mask)
+{
+        u32 enable_mask;
+
+        if (IS_VALLEYVIEW(dev_priv->dev))
+                enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
+                                                           status_mask);
+        else
+                enable_mask = status_mask << 16;
+        __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
+}
+
+void
+i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
+                      u32 status_mask)
+{
+        u32 enable_mask;
+
+        if (IS_VALLEYVIEW(dev_priv->dev))
+                enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
+                                                           status_mask);
+        else
+                enable_mask = status_mask << 16;
+        __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
+}
+
 /**
  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
  */
 static void i915_enable_asle_pipestat(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
 
         if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
@@ -504,10 +604,10 @@ static void i915_enable_asle_pipestat(struct drm_device *dev)
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 
-        i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_ENABLE);
+        i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
         if (INTEL_INFO(dev)->gen >= 4)
                 i915_enable_pipestat(dev_priv, PIPE_A,
-                                     PIPE_LEGACY_BLC_EVENT_ENABLE);
+                                     PIPE_LEGACY_BLC_EVENT_STATUS);
 
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
@@ -524,7 +624,7 @@ static void i915_enable_asle_pipestat(struct drm_device *dev)
 static int
 i915_pipe_enabled(struct drm_device *dev, int pipe)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
                 /* Locking is horribly broken here, but whatever. */
@@ -548,7 +648,7 @@ static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
  */
 static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long high_frame;
         unsigned long low_frame;
         u32 high1, high2, low, pixel, vbl_start;
@@ -604,7 +704,7 @@ static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
 
 static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int reg = PIPE_FRMCOUNT_GM45(pipe);
 
         if (!i915_pipe_enabled(dev, pipe)) {
@@ -859,8 +959,8 @@ static bool intel_hpd_irq_event(struct drm_device *dev,
 
 static void i915_hotplug_work_func(struct work_struct *work)
 {
-        drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
-                                                    hotplug_work);
+        struct drm_i915_private *dev_priv =
+                container_of(work, struct drm_i915_private, hotplug_work);
         struct drm_device *dev = dev_priv->dev;
         struct drm_mode_config *mode_config = &dev->mode_config;
         struct intel_connector *intel_connector;
@@ -928,9 +1028,14 @@ static void i915_hotplug_work_func(struct work_struct *work)
                 drm_kms_helper_hotplug_event(dev);
 }
 
+static void intel_hpd_irq_uninstall(struct drm_i915_private *dev_priv)
+{
+        del_timer_sync(&dev_priv->hotplug_reenable_timer);
+}
+
 static void ironlake_rps_change_irq_handler(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 busy_up, busy_down, max_avg, min_avg;
         u8 new_delay;
 
@@ -981,8 +1086,8 @@ static void notify_ring(struct drm_device *dev,
 
 static void gen6_pm_rps_work(struct work_struct *work)
 {
-        drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
-                                                    rps.work);
+        struct drm_i915_private *dev_priv =
+                container_of(work, struct drm_i915_private, rps.work);
         u32 pm_iir;
         int new_delay, adj;
 
@@ -990,13 +1095,13 @@ static void gen6_pm_rps_work(struct work_struct *work)
         pm_iir = dev_priv->rps.pm_iir;
         dev_priv->rps.pm_iir = 0;
         /* Make sure not to corrupt PMIMR state used by ringbuffer code */
-        snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS);
+        snb_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
         spin_unlock_irq(&dev_priv->irq_lock);
 
         /* Make sure we didn't queue anything we're not going to process. */
-        WARN_ON(pm_iir & ~GEN6_PM_RPS_EVENTS);
+        WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
 
-        if ((pm_iir & GEN6_PM_RPS_EVENTS) == 0)
+        if ((pm_iir & dev_priv->pm_rps_events) == 0)
                 return;
 
         mutex_lock(&dev_priv->rps.hw_lock);
@@ -1007,36 +1112,38 @@ static void gen6_pm_rps_work(struct work_struct *work)
                         adj *= 2;
                 else
                         adj = 1;
-                new_delay = dev_priv->rps.cur_delay + adj;
+                new_delay = dev_priv->rps.cur_freq + adj;
 
                 /*
                  * For better performance, jump directly
                  * to RPe if we're below it.
                  */
-                if (new_delay < dev_priv->rps.rpe_delay)
-                        new_delay = dev_priv->rps.rpe_delay;
+                if (new_delay < dev_priv->rps.efficient_freq)
+                        new_delay = dev_priv->rps.efficient_freq;
         } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
-                if (dev_priv->rps.cur_delay > dev_priv->rps.rpe_delay)
-                        new_delay = dev_priv->rps.rpe_delay;
+                if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
+                        new_delay = dev_priv->rps.efficient_freq;
                 else
-                        new_delay = dev_priv->rps.min_delay;
+                        new_delay = dev_priv->rps.min_freq_softlimit;
                 adj = 0;
         } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
                 if (adj < 0)
                         adj *= 2;
                 else
                         adj = -1;
-                new_delay = dev_priv->rps.cur_delay + adj;
+                new_delay = dev_priv->rps.cur_freq + adj;
         } else { /* unknown event */
-                new_delay = dev_priv->rps.cur_delay;
+                new_delay = dev_priv->rps.cur_freq;
         }
 
         /* sysfs frequency interfaces may have snuck in while servicing the
          * interrupt
          */
         new_delay = clamp_t(int, new_delay,
-                            dev_priv->rps.min_delay, dev_priv->rps.max_delay);
-        dev_priv->rps.last_adj = new_delay - dev_priv->rps.cur_delay;
+                            dev_priv->rps.min_freq_softlimit,
+                            dev_priv->rps.max_freq_softlimit);
+
+        dev_priv->rps.last_adj = new_delay - dev_priv->rps.cur_freq;
 
         if (IS_VALLEYVIEW(dev_priv->dev))
                 valleyview_set_rps(dev_priv->dev, new_delay);
@@ -1058,8 +1165,8 @@ static void gen6_pm_rps_work(struct work_struct *work)
  */
 static void ivybridge_parity_work(struct work_struct *work)
 {
-        drm_i915_private_t *dev_priv = container_of(work, drm_i915_private_t,
-                                                    l3_parity.error_work);
+        struct drm_i915_private *dev_priv =
+                container_of(work, struct drm_i915_private, l3_parity.error_work);
         u32 error_status, row, bank, subbank;
         char *parity_event[6];
         uint32_t misccpctl;
@@ -1131,7 +1238,7 @@ out:
 
 static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         if (!HAS_L3_DPF(dev))
                 return;
@@ -1177,8 +1284,8 @@ static void snb_gt_irq_handler(struct drm_device *dev,
         if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
                       GT_BSD_CS_ERROR_INTERRUPT |
                       GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) {
-                DRM_ERROR("GT error interrupt 0x%08x\n", gt_iir);
-                i915_handle_error(dev, false);
+                i915_handle_error(dev, false, "GT error interrupt 0x%08x",
+                                  gt_iir);
         }
 
         if (gt_iir & GT_PARITY_ERROR(dev))
@@ -1242,13 +1349,16 @@ static inline void intel_hpd_irq_handler(struct drm_device *dev,
                                          u32 hotplug_trigger,
                                          const u32 *hpd)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int i;
         bool storm_detected = false;
 
         if (!hotplug_trigger)
                 return;
 
+        DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
+                          hotplug_trigger);
+
         spin_lock(&dev_priv->irq_lock);
         for (i = 1; i < HPD_NUM_PINS; i++) {
 
@@ -1295,14 +1405,14 @@ static inline void intel_hpd_irq_handler(struct drm_device *dev,
 
 static void gmbus_irq_handler(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         wake_up_all(&dev_priv->gmbus_wait_queue);
 }
 
 static void dp_aux_irq_handler(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         wake_up_all(&dev_priv->gmbus_wait_queue);
 }
@@ -1408,10 +1518,10 @@ static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
  * the work queue. */
 static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
 {
-        if (pm_iir & GEN6_PM_RPS_EVENTS) {
+        if (pm_iir & dev_priv->pm_rps_events) {
                 spin_lock(&dev_priv->irq_lock);
-                dev_priv->rps.pm_iir |= pm_iir & GEN6_PM_RPS_EVENTS;
-                snb_disable_pm_irq(dev_priv, pm_iir & GEN6_PM_RPS_EVENTS);
+                dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
+                snb_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
                 spin_unlock(&dev_priv->irq_lock);
 
                 queue_work(dev_priv->wq, &dev_priv->rps.work);
@@ -1422,23 +1532,89 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
                         notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
 
                 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) {
-                        DRM_ERROR("VEBOX CS error interrupt 0x%08x\n", pm_iir);
-                        i915_handle_error(dev_priv->dev, false);
+                        i915_handle_error(dev_priv->dev, false,
+                                          "VEBOX CS error interrupt 0x%08x",
+                                          pm_iir);
                 }
         }
 }
 
+static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 pipe_stats[I915_MAX_PIPES] = { };
+        int pipe;
+
+        spin_lock(&dev_priv->irq_lock);
+        for_each_pipe(pipe) {
+                int reg;
+                u32 mask, iir_bit = 0;
+
+                /*
+                 * PIPESTAT bits get signalled even when the interrupt is
+                 * disabled with the mask bits, and some of the status bits do
+                 * not generate interrupts at all (like the underrun bit). Hence
+                 * we need to be careful that we only handle what we want to
+                 * handle.
+                 */
+                mask = 0;
+                if (__cpu_fifo_underrun_reporting_enabled(dev, pipe))
+                        mask |= PIPE_FIFO_UNDERRUN_STATUS;
+
+                switch (pipe) {
+                case PIPE_A:
+                        iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
+                        break;
+                case PIPE_B:
+                        iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
+                        break;
+                }
+                if (iir & iir_bit)
+                        mask |= dev_priv->pipestat_irq_mask[pipe];
+
+                if (!mask)
+                        continue;
+
+                reg = PIPESTAT(pipe);
+                mask |= PIPESTAT_INT_ENABLE_MASK;
+                pipe_stats[pipe] = I915_READ(reg) & mask;
+
+                /*
+                 * Clear the PIPE*STAT regs before the IIR
+                 */
+                if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
+                                        PIPESTAT_INT_STATUS_MASK))
+                        I915_WRITE(reg, pipe_stats[pipe]);
+        }
+        spin_unlock(&dev_priv->irq_lock);
+
+        for_each_pipe(pipe) {
+                if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
+                        drm_handle_vblank(dev, pipe);
+
+                if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
+                        intel_prepare_page_flip(dev, pipe);
+                        intel_finish_page_flip(dev, pipe);
+                }
+
+                if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
+                        i9xx_pipe_crc_irq_handler(dev, pipe);
+
+                if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
+                    intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
+                        DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
+        }
+
+        if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
+                gmbus_irq_handler(dev);
+}
+
 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = (struct drm_device *) arg;
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 iir, gt_iir, pm_iir;
         irqreturn_t ret = IRQ_NONE;
-        unsigned long irqflags;
-        int pipe;
-        u32 pipe_stats[I915_MAX_PIPES];
-
-        atomic_inc(&dev_priv->irq_received);
 
         while (true) {
                 iir = I915_READ(VLV_IIR);
@@ -1452,44 +1628,13 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 
                 snb_gt_irq_handler(dev, dev_priv, gt_iir);
 
-                spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-                for_each_pipe(pipe) {
-                        int reg = PIPESTAT(pipe);
-                        pipe_stats[pipe] = I915_READ(reg);
-
-                        /*
-                         * Clear the PIPE*STAT regs before the IIR
-                         */
-                        if (pipe_stats[pipe] & 0x8000ffff) {
-                                if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
-                                        DRM_DEBUG_DRIVER("pipe %c underrun\n",
-                                                         pipe_name(pipe));
-                                I915_WRITE(reg, pipe_stats[pipe]);
-                        }
-                }
-                spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
-
-                for_each_pipe(pipe) {
-                        if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
-                                drm_handle_vblank(dev, pipe);
-
-                        if (pipe_stats[pipe] & PLANE_FLIPDONE_INT_STATUS_VLV) {
-                                intel_prepare_page_flip(dev, pipe);
-                                intel_finish_page_flip(dev, pipe);
-                        }
-
-                        if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
-                                i9xx_pipe_crc_irq_handler(dev, pipe);
-                }
+                valleyview_pipestat_irq_handler(dev, iir);
 
                 /* Consume port.  Then clear IIR or we'll miss events */
                 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
                         u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
 
-                        DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
-                                         hotplug_status);
-
                         intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
 
                         if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
@@ -1499,8 +1644,6 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
                         I915_READ(PORT_HOTPLUG_STAT);
                 }
 
-                if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
-                        gmbus_irq_handler(dev);
 
                 if (pm_iir)
                         gen6_rps_irq_handler(dev_priv, pm_iir);
@@ -1516,7 +1659,7 @@ out:
 
 static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
 
@@ -1559,12 +1702,12 @@ static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
         if (pch_iir & SDE_TRANSA_FIFO_UNDER)
                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
                                                           false))
-                        DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
+                        DRM_ERROR("PCH transcoder A FIFO underrun\n");
 
         if (pch_iir & SDE_TRANSB_FIFO_UNDER)
                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
                                                           false))
-                        DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
+                        DRM_ERROR("PCH transcoder B FIFO underrun\n");
 }
 
 static void ivb_err_int_handler(struct drm_device *dev)
@@ -1580,8 +1723,8 @@ static void ivb_err_int_handler(struct drm_device *dev)
                 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe)) {
                         if (intel_set_cpu_fifo_underrun_reporting(dev, pipe,
                                                                   false))
-                                DRM_DEBUG_DRIVER("Pipe %c FIFO underrun\n",
-                                                 pipe_name(pipe));
+                                DRM_ERROR("Pipe %c FIFO underrun\n",
+                                          pipe_name(pipe));
                 }
 
                 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
@@ -1606,24 +1749,24 @@ static void cpt_serr_int_handler(struct drm_device *dev)
         if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A,
                                                           false))
-                        DRM_DEBUG_DRIVER("PCH transcoder A FIFO underrun\n");
+                        DRM_ERROR("PCH transcoder A FIFO underrun\n");
 
         if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_B,
                                                           false))
-                        DRM_DEBUG_DRIVER("PCH transcoder B FIFO underrun\n");
+                        DRM_ERROR("PCH transcoder B FIFO underrun\n");
 
         if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
                 if (intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_C,
                                                           false))
-                        DRM_DEBUG_DRIVER("PCH transcoder C FIFO underrun\n");
+                        DRM_ERROR("PCH transcoder C FIFO underrun\n");
 
         I915_WRITE(SERR_INT, serr_int);
 }
 
 static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
 
@@ -1678,8 +1821,8 @@ static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
 
                 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
                         if (intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
-                                DRM_DEBUG_DRIVER("Pipe %c FIFO underrun\n",
-                                                 pipe_name(pipe));
+                                DRM_ERROR("Pipe %c FIFO underrun\n",
+                                          pipe_name(pipe));
 
                 if (de_iir & DE_PIPE_CRC_DONE(pipe))
                         i9xx_pipe_crc_irq_handler(dev, pipe);
@@ -1711,7 +1854,7 @@ static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
 static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        enum pipe i;
+        enum pipe pipe;
 
         if (de_iir & DE_ERR_INT_IVB)
                 ivb_err_int_handler(dev);
@@ -1722,14 +1865,14 @@ static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
         if (de_iir & DE_GSE_IVB)
                 intel_opregion_asle_intr(dev);
 
-        for_each_pipe(i) {
-                if (de_iir & (DE_PIPE_VBLANK_IVB(i)))
-                        drm_handle_vblank(dev, i);
+        for_each_pipe(pipe) {
+                if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
+                        drm_handle_vblank(dev, pipe);
 
                 /* plane/pipes map 1:1 on ilk+ */
-                if (de_iir & DE_PLANE_FLIP_DONE_IVB(i)) {
-                        intel_prepare_page_flip(dev, i);
-                        intel_finish_page_flip_plane(dev, i);
+                if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
+                        intel_prepare_page_flip(dev, pipe);
+                        intel_finish_page_flip_plane(dev, pipe);
                 }
         }
 
@@ -1747,12 +1890,10 @@ static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
 static irqreturn_t ironlake_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = (struct drm_device *) arg;
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 de_iir, gt_iir, de_ier, sde_ier = 0;
         irqreturn_t ret = IRQ_NONE;
 
-        atomic_inc(&dev_priv->irq_received);
-
         /* We get interrupts on unclaimed registers, so check for this before we
          * do any I915_{READ,WRITE}. */
         intel_uncore_check_errors(dev);
@@ -1821,8 +1962,6 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg)
         uint32_t tmp = 0;
         enum pipe pipe;
 
-        atomic_inc(&dev_priv->irq_received);
-
         master_ctl = I915_READ(GEN8_MASTER_IRQ);
         master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
         if (!master_ctl)
@@ -1884,8 +2023,8 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg)
                 if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN) {
                         if (intel_set_cpu_fifo_underrun_reporting(dev, pipe,
                                                                   false))
-                                DRM_DEBUG_DRIVER("Pipe %c FIFO underrun\n",
-                                                 pipe_name(pipe));
+                                DRM_ERROR("Pipe %c FIFO underrun\n",
+                                          pipe_name(pipe));
                 }
 
                 if (pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS) {
@@ -1962,8 +2101,8 @@ static void i915_error_work_func(struct work_struct *work)
 {
         struct i915_gpu_error *error = container_of(work, struct i915_gpu_error,
                                                     work);
-        drm_i915_private_t *dev_priv = container_of(error, drm_i915_private_t,
-                                                    gpu_error);
+        struct drm_i915_private *dev_priv =
+                container_of(error, struct drm_i915_private, gpu_error);
         struct drm_device *dev = dev_priv->dev;
         char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
         char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
@@ -2127,11 +2266,18 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
  * so userspace knows something bad happened (should trigger collection
  * of a ring dump etc.).
  */
-void i915_handle_error(struct drm_device *dev, bool wedged)
+void i915_handle_error(struct drm_device *dev, bool wedged,
+                       const char *fmt, ...)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        va_list args;
+        char error_msg[80];
 
-        i915_capture_error_state(dev);
+        va_start(args, fmt);
+        vscnprintf(error_msg, sizeof(error_msg), fmt, args);
+        va_end(args);
+
+        i915_capture_error_state(dev, wedged, error_msg);
         i915_report_and_clear_eir(dev);
 
         if (wedged) {
@@ -2165,7 +2311,7 @@ void i915_handle_error(struct drm_device *dev, bool wedged)
 
 static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct drm_i915_gem_object *obj;
@@ -2197,8 +2343,8 @@ static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, in
         } else {
                 int dspaddr = DSPADDR(intel_crtc->plane);
                 stall_detected = I915_READ(dspaddr) == (i915_gem_obj_ggtt_offset(obj) +
-                                                        crtc->y * crtc->fb->pitches[0] +
-                                                        crtc->x * crtc->fb->bits_per_pixel/8);
+                                                        crtc->y * crtc->primary->fb->pitches[0] +
+                                                        crtc->x * crtc->primary->fb->bits_per_pixel/8);
         }
 
         spin_unlock_irqrestore(&dev->event_lock, flags);
@@ -2214,7 +2360,7 @@ static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, in
  */
 static int i915_enable_vblank(struct drm_device *dev, int pipe)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
 
         if (!i915_pipe_enabled(dev, pipe))
@@ -2223,13 +2369,13 @@ static int i915_enable_vblank(struct drm_device *dev, int pipe)
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
         if (INTEL_INFO(dev)->gen >= 4)
                 i915_enable_pipestat(dev_priv, pipe,
-                                     PIPE_START_VBLANK_INTERRUPT_ENABLE);
+                                     PIPE_START_VBLANK_INTERRUPT_STATUS);
         else
                 i915_enable_pipestat(dev_priv, pipe,
-                                     PIPE_VBLANK_INTERRUPT_ENABLE);
+                                     PIPE_VBLANK_INTERRUPT_STATUS);
 
         /* maintain vblank delivery even in deep C-states */
-        if (dev_priv->info->gen == 3)
+        if (INTEL_INFO(dev)->gen == 3)
                 I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_AGPBUSY_DIS));
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
@@ -2238,7 +2384,7 @@ static int i915_enable_vblank(struct drm_device *dev, int pipe)
 
 static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
                                                      DE_PIPE_VBLANK(pipe);
@@ -2255,22 +2401,15 @@ static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
 
 static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
-        u32 imr;
 
         if (!i915_pipe_enabled(dev, pipe))
                 return -EINVAL;
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-        imr = I915_READ(VLV_IMR);
-        if (pipe == PIPE_A)
-                imr &= ~I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
-        else
-                imr &= ~I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
-        I915_WRITE(VLV_IMR, imr);
         i915_enable_pipestat(dev_priv, pipe,
-                             PIPE_START_VBLANK_INTERRUPT_ENABLE);
+                             PIPE_START_VBLANK_INTERRUPT_STATUS);
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
         return 0;
@@ -2297,22 +2436,22 @@ static int gen8_enable_vblank(struct drm_device *dev, int pipe)
  */
 static void i915_disable_vblank(struct drm_device *dev, int pipe)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-        if (dev_priv->info->gen == 3)
+        if (INTEL_INFO(dev)->gen == 3)
                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_DIS));
 
         i915_disable_pipestat(dev_priv, pipe,
-                              PIPE_VBLANK_INTERRUPT_ENABLE |
-                              PIPE_START_VBLANK_INTERRUPT_ENABLE);
+                              PIPE_VBLANK_INTERRUPT_STATUS |
+                              PIPE_START_VBLANK_INTERRUPT_STATUS);
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
 static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
                                                      DE_PIPE_VBLANK(pipe);
@@ -2324,19 +2463,12 @@ static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
 
 static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
-        u32 imr;
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
         i915_disable_pipestat(dev_priv, pipe,
-                              PIPE_START_VBLANK_INTERRUPT_ENABLE);
-        imr = I915_READ(VLV_IMR);
-        if (pipe == PIPE_A)
-                imr |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
-        else
-                imr |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
-        I915_WRITE(VLV_IMR, imr);
+                              PIPE_START_VBLANK_INTERRUPT_STATUS);
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
@@ -2373,29 +2505,43 @@ static struct intel_ring_buffer *
 semaphore_waits_for(struct intel_ring_buffer *ring, u32 *seqno)
 {
         struct drm_i915_private *dev_priv = ring->dev->dev_private;
-        u32 cmd, ipehr, acthd, acthd_min;
+        u32 cmd, ipehr, head;
+        int i;
 
         ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
         if ((ipehr & ~(0x3 << 16)) !=
             (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE | MI_SEMAPHORE_REGISTER))
                 return NULL;
 
-        /* ACTHD is likely pointing to the dword after the actual command,
-         * so scan backwards until we find the MBOX.
+        /*
+         * HEAD is likely pointing to the dword after the actual command,
+         * so scan backwards until we find the MBOX. But limit it to just 3
+         * dwords. Note that we don't care about ACTHD here since that might
+         * point at at batch, and semaphores are always emitted into the
+         * ringbuffer itself.
          */
-        acthd = intel_ring_get_active_head(ring) & HEAD_ADDR;
-        acthd_min = max((int)acthd - 3 * 4, 0);
-        do {
-                cmd = ioread32(ring->virtual_start + acthd);
+        head = I915_READ_HEAD(ring) & HEAD_ADDR;
+
+        for (i = 4; i; --i) {
+                /*
+                 * Be paranoid and presume the hw has gone off into the wild -
+                 * our ring is smaller than what the hardware (and hence
+                 * HEAD_ADDR) allows. Also handles wrap-around.
+                 */
+                head &= ring->size - 1;
+
+                /* This here seems to blow up */
+                cmd = ioread32(ring->virtual_start + head);
                 if (cmd == ipehr)
                         break;
 
-                acthd -= 4;
-                if (acthd < acthd_min)
-                        return NULL;
-        } while (1);
+                head -= 4;
+        }
 
-        *seqno = ioread32(ring->virtual_start+acthd+4)+1;
+        if (!i)
+                return NULL;
+
+        *seqno = ioread32(ring->virtual_start + head + 4) + 1;
         return &dev_priv->ring[(ring->id + (((ipehr >> 17) & 1) + 1)) % 3];
 }
 
@@ -2429,7 +2575,7 @@ static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
 }
 
 static enum intel_ring_hangcheck_action
-ring_stuck(struct intel_ring_buffer *ring, u32 acthd)
+ring_stuck(struct intel_ring_buffer *ring, u64 acthd)
 {
         struct drm_device *dev = ring->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -2448,9 +2594,9 @@ ring_stuck(struct intel_ring_buffer *ring, u32 acthd)
          */
         tmp = I915_READ_CTL(ring);
         if (tmp & RING_WAIT) {
-                DRM_ERROR("Kicking stuck wait on %s\n",
-                          ring->name);
-                i915_handle_error(dev, false);
+                i915_handle_error(dev, false,
+                                  "Kicking stuck wait on %s",
+                                  ring->name);
                 I915_WRITE_CTL(ring, tmp);
                 return HANGCHECK_KICK;
         }
@@ -2460,9 +2606,9 @@ ring_stuck(struct intel_ring_buffer *ring, u32 acthd)
                 default:
                         return HANGCHECK_HUNG;
                 case 1:
-                        DRM_ERROR("Kicking stuck semaphore on %s\n",
-                                  ring->name);
-                        i915_handle_error(dev, false);
+                        i915_handle_error(dev, false,
+                                          "Kicking stuck semaphore on %s",
+                                          ring->name);
                         I915_WRITE_CTL(ring, tmp);
                         return HANGCHECK_KICK;
                 case 0:
@@ -2484,7 +2630,7 @@ ring_stuck(struct intel_ring_buffer *ring, u32 acthd)
 static void i915_hangcheck_elapsed(unsigned long data)
 {
         struct drm_device *dev = (struct drm_device *)data;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring;
         int i;
         int busy_count = 0, rings_hung = 0;
@@ -2492,13 +2638,13 @@ static void i915_hangcheck_elapsed(unsigned long data)
 #define BUSY 1
 #define KICK 5
 #define HUNG 20
-#define FIRE 30
 
-        if (!i915_enable_hangcheck)
+        if (!i915.enable_hangcheck)
                 return;
 
         for_each_ring(ring, dev_priv, i) {
-                u32 seqno, acthd;
+                u64 acthd;
+                u32 seqno;
                 bool busy = true;
 
                 semaphore_clear_deadlocks(dev_priv);
@@ -2576,7 +2722,7 @@ static void i915_hangcheck_elapsed(unsigned long data)
         }
 
         for_each_ring(ring, dev_priv, i) {
-                if (ring->hangcheck.score > FIRE) {
+                if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
                         DRM_INFO("%s on %s\n",
                                  stuck[i] ? "stuck" : "no progress",
                                  ring->name);
@@ -2585,7 +2731,7 @@ static void i915_hangcheck_elapsed(unsigned long data)
         }
 
         if (rings_hung)
-                return i915_handle_error(dev, true);
+                return i915_handle_error(dev, true, "Ring hung");
 
         if (busy_count)
                 /* Reset timer case chip hangs without another request
@@ -2596,7 +2742,7 @@ static void i915_hangcheck_elapsed(unsigned long data)
 void i915_queue_hangcheck(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        if (!i915_enable_hangcheck)
+        if (!i915.enable_hangcheck)
                 return;
 
         mod_timer(&dev_priv->gpu_error.hangcheck_timer,
@@ -2643,9 +2789,7 @@ static void gen5_gt_irq_preinstall(struct drm_device *dev)
 */
 static void ironlake_irq_preinstall(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-
-        atomic_set(&dev_priv->irq_received, 0);
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         I915_WRITE(HWSTAM, 0xeffe);
 
@@ -2660,11 +2804,9 @@ static void ironlake_irq_preinstall(struct drm_device *dev)
 
 static void valleyview_irq_preinstall(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
 
-        atomic_set(&dev_priv->irq_received, 0);
-
         /* VLV magic */
         I915_WRITE(VLV_IMR, 0);
         I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
@@ -2694,8 +2836,6 @@ static void gen8_irq_preinstall(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
 
-        atomic_set(&dev_priv->irq_received, 0);
-
         I915_WRITE(GEN8_MASTER_IRQ, 0);
         POSTING_READ(GEN8_MASTER_IRQ);
 
@@ -2740,7 +2880,7 @@ static void gen8_irq_preinstall(struct drm_device *dev)
 
 static void ibx_hpd_irq_setup(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_mode_config *mode_config = &dev->mode_config;
         struct intel_encoder *intel_encoder;
         u32 hotplug_irqs, hotplug, enabled_irqs = 0;
@@ -2775,7 +2915,7 @@ static void ibx_hpd_irq_setup(struct drm_device *dev)
 
 static void ibx_irq_postinstall(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 mask;
 
         if (HAS_PCH_NOP(dev))
@@ -2821,7 +2961,7 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev)
         POSTING_READ(GTIER);
 
         if (INTEL_INFO(dev)->gen >= 6) {
-                pm_irqs |= GEN6_PM_RPS_EVENTS;
+                pm_irqs |= dev_priv->pm_rps_events;
 
                 if (HAS_VEBOX(dev))
                         pm_irqs |= PM_VEBOX_USER_INTERRUPT;
@@ -2837,7 +2977,7 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev)
 static int ironlake_irq_postinstall(struct drm_device *dev)
 {
         unsigned long irqflags;
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 display_mask, extra_mask;
 
         if (INTEL_INFO(dev)->gen >= 7) {
@@ -2885,44 +3025,113 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
         return 0;
 }
 
+static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
+{
+        u32 pipestat_mask;
+        u32 iir_mask;
+
+        pipestat_mask = PIPESTAT_INT_STATUS_MASK |
+                        PIPE_FIFO_UNDERRUN_STATUS;
+
+        I915_WRITE(PIPESTAT(PIPE_A), pipestat_mask);
+        I915_WRITE(PIPESTAT(PIPE_B), pipestat_mask);
+        POSTING_READ(PIPESTAT(PIPE_A));
+
+        pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
+                        PIPE_CRC_DONE_INTERRUPT_STATUS;
+
+        i915_enable_pipestat(dev_priv, PIPE_A, pipestat_mask |
+                                               PIPE_GMBUS_INTERRUPT_STATUS);
+        i915_enable_pipestat(dev_priv, PIPE_B, pipestat_mask);
+
+        iir_mask = I915_DISPLAY_PORT_INTERRUPT |
+                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
+                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
+        dev_priv->irq_mask &= ~iir_mask;
+
+        I915_WRITE(VLV_IIR, iir_mask);
+        I915_WRITE(VLV_IIR, iir_mask);
+        I915_WRITE(VLV_IMR, dev_priv->irq_mask);
+        I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
+        POSTING_READ(VLV_IER);
+}
+
+static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
+{
+        u32 pipestat_mask;
+        u32 iir_mask;
+
+        iir_mask = I915_DISPLAY_PORT_INTERRUPT |
+                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
+                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
+
+        dev_priv->irq_mask |= iir_mask;
+        I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
+        I915_WRITE(VLV_IMR, dev_priv->irq_mask);
+        I915_WRITE(VLV_IIR, iir_mask);
+        I915_WRITE(VLV_IIR, iir_mask);
+        POSTING_READ(VLV_IIR);
+
+        pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
+                        PIPE_CRC_DONE_INTERRUPT_STATUS;
+
+        i915_disable_pipestat(dev_priv, PIPE_A, pipestat_mask |
+                                                PIPE_GMBUS_INTERRUPT_STATUS);
+        i915_disable_pipestat(dev_priv, PIPE_B, pipestat_mask);
+
+        pipestat_mask = PIPESTAT_INT_STATUS_MASK |
+                        PIPE_FIFO_UNDERRUN_STATUS;
+        I915_WRITE(PIPESTAT(PIPE_A), pipestat_mask);
+        I915_WRITE(PIPESTAT(PIPE_B), pipestat_mask);
+        POSTING_READ(PIPESTAT(PIPE_A));
+}
+
+void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
+{
+        assert_spin_locked(&dev_priv->irq_lock);
+
+        if (dev_priv->display_irqs_enabled)
+                return;
+
+        dev_priv->display_irqs_enabled = true;
+
+        if (dev_priv->dev->irq_enabled)
+                valleyview_display_irqs_install(dev_priv);
+}
+
+void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
+{
+        assert_spin_locked(&dev_priv->irq_lock);
+
+        if (!dev_priv->display_irqs_enabled)
+                return;
+
+        dev_priv->display_irqs_enabled = false;
+
+        if (dev_priv->dev->irq_enabled)
+                valleyview_display_irqs_uninstall(dev_priv);
+}
+
 static int valleyview_irq_postinstall(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-        u32 enable_mask;
-        u32 pipestat_enable = PLANE_FLIP_DONE_INT_EN_VLV |
-                PIPE_CRC_DONE_ENABLE;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
 
-        enable_mask = I915_DISPLAY_PORT_INTERRUPT;
-        enable_mask |= I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
-                I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
-                I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
-                I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
-
-        /*
-         *Leave vblank interrupts masked initially.  enable/disable will
-         * toggle them based on usage.
-         */
-        dev_priv->irq_mask = (~enable_mask) |
-                I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
-                I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
+        dev_priv->irq_mask = ~0;
 
         I915_WRITE(PORT_HOTPLUG_EN, 0);
         POSTING_READ(PORT_HOTPLUG_EN);
 
         I915_WRITE(VLV_IMR, dev_priv->irq_mask);
-        I915_WRITE(VLV_IER, enable_mask);
+        I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
         I915_WRITE(VLV_IIR, 0xffffffff);
-        I915_WRITE(PIPESTAT(0), 0xffff);
-        I915_WRITE(PIPESTAT(1), 0xffff);
         POSTING_READ(VLV_IER);
 
         /* Interrupt setup is already guaranteed to be single-threaded, this is
          * just to make the assert_spin_locked check happy. */
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-        i915_enable_pipestat(dev_priv, PIPE_A, pipestat_enable);
-        i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_EVENT_ENABLE);
-        i915_enable_pipestat(dev_priv, PIPE_B, pipestat_enable);
+        if (dev_priv->display_irqs_enabled)
+                valleyview_display_irqs_install(dev_priv);
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
         I915_WRITE(VLV_IIR, 0xffffffff);
@@ -3018,8 +3227,6 @@ static void gen8_irq_uninstall(struct drm_device *dev)
         if (!dev_priv)
                 return;
 
-        atomic_set(&dev_priv->irq_received, 0);
-
         I915_WRITE(GEN8_MASTER_IRQ, 0);
 
 #define GEN8_IRQ_FINI_NDX(type, which) do { \
@@ -3054,13 +3261,14 @@ static void gen8_irq_uninstall(struct drm_device *dev)
 
 static void valleyview_irq_uninstall(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long irqflags;
         int pipe;
 
         if (!dev_priv)
                 return;
 
-        del_timer_sync(&dev_priv->hotplug_reenable_timer);
+        intel_hpd_irq_uninstall(dev_priv);
 
         for_each_pipe(pipe)
                 I915_WRITE(PIPESTAT(pipe), 0xffff);
@@ -3068,8 +3276,14 @@ static void valleyview_irq_uninstall(struct drm_device *dev)
         I915_WRITE(HWSTAM, 0xffffffff);
         I915_WRITE(PORT_HOTPLUG_EN, 0);
         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
-        for_each_pipe(pipe)
-                I915_WRITE(PIPESTAT(pipe), 0xffff);
+
+        spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
+        if (dev_priv->display_irqs_enabled)
+                valleyview_display_irqs_uninstall(dev_priv);
+        spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
+
+        dev_priv->irq_mask = 0;
+
         I915_WRITE(VLV_IIR, 0xffffffff);
         I915_WRITE(VLV_IMR, 0xffffffff);
         I915_WRITE(VLV_IER, 0x0);
@@ -3078,12 +3292,12 @@ static void valleyview_irq_uninstall(struct drm_device *dev)
 
 static void ironlake_irq_uninstall(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         if (!dev_priv)
                 return;
 
-        del_timer_sync(&dev_priv->hotplug_reenable_timer);
+        intel_hpd_irq_uninstall(dev_priv);
 
         I915_WRITE(HWSTAM, 0xffffffff);
 
@@ -3109,11 +3323,9 @@ static void ironlake_irq_uninstall(struct drm_device *dev)
 
 static void i8xx_irq_preinstall(struct drm_device * dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
 
-        atomic_set(&dev_priv->irq_received, 0);
-
         for_each_pipe(pipe)
                 I915_WRITE(PIPESTAT(pipe), 0);
         I915_WRITE16(IMR, 0xffff);
@@ -3123,7 +3335,7 @@ static void i8xx_irq_preinstall(struct drm_device * dev)
 
 static int i8xx_irq_postinstall(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
 
         I915_WRITE16(EMR,
@@ -3148,8 +3360,8 @@ static int i8xx_irq_postinstall(struct drm_device *dev)
         /* Interrupt setup is already guaranteed to be single-threaded, this is
          * just to make the assert_spin_locked check happy. */
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-        i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_ENABLE);
-        i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_ENABLE);
+        i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
+        i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
         return 0;
@@ -3161,7 +3373,7 @@ static int i8xx_irq_postinstall(struct drm_device *dev)
 static bool i8xx_handle_vblank(struct drm_device *dev,
                                int plane, int pipe, u32 iir)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
 
         if (!drm_handle_vblank(dev, pipe))
@@ -3189,7 +3401,7 @@ static bool i8xx_handle_vblank(struct drm_device *dev,
 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = (struct drm_device *) arg;
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u16 iir, new_iir;
         u32 pipe_stats[2];
         unsigned long irqflags;
@@ -3198,8 +3410,6 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
 
-        atomic_inc(&dev_priv->irq_received);
-
         iir = I915_READ16(IIR);
         if (iir == 0)
                 return IRQ_NONE;
@@ -3212,7 +3422,9 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
                  */
                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
-                        i915_handle_error(dev, false);
+                        i915_handle_error(dev, false,
+                                          "Command parser error, iir 0x%08x",
+                                          iir);
 
                 for_each_pipe(pipe) {
                         int reg = PIPESTAT(pipe);
@@ -3221,12 +3433,8 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
                         /*
                          * Clear the PIPE*STAT regs before the IIR
                          */
-                        if (pipe_stats[pipe] & 0x8000ffff) {
-                                if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
-                                        DRM_DEBUG_DRIVER("pipe %c underrun\n",
-                                                         pipe_name(pipe));
+                        if (pipe_stats[pipe] & 0x8000ffff)
                                 I915_WRITE(reg, pipe_stats[pipe]);
-                        }
                 }
                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
@@ -3249,6 +3457,10 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 
                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
                                 i9xx_pipe_crc_irq_handler(dev, pipe);
+
+                        if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
+                            intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
+                                DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
                 }
 
                 iir = new_iir;
@@ -3259,7 +3471,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 
 static void i8xx_irq_uninstall(struct drm_device * dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
 
         for_each_pipe(pipe) {
@@ -3274,11 +3486,9 @@ static void i8xx_irq_uninstall(struct drm_device * dev)
 
 static void i915_irq_preinstall(struct drm_device * dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
 
-        atomic_set(&dev_priv->irq_received, 0);
-
         if (I915_HAS_HOTPLUG(dev)) {
                 I915_WRITE(PORT_HOTPLUG_EN, 0);
                 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
@@ -3294,7 +3504,7 @@ static void i915_irq_preinstall(struct drm_device * dev)
 
 static int i915_irq_postinstall(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 enable_mask;
         unsigned long irqflags;
 
@@ -3335,8 +3545,8 @@ static int i915_irq_postinstall(struct drm_device *dev)
         /* Interrupt setup is already guaranteed to be single-threaded, this is
          * just to make the assert_spin_locked check happy. */
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-        i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_ENABLE);
-        i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_ENABLE);
+        i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
+        i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
         return 0;
@@ -3348,7 +3558,7 @@ static int i915_irq_postinstall(struct drm_device *dev)
 static bool i915_handle_vblank(struct drm_device *dev,
                                int plane, int pipe, u32 iir)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
 
         if (!drm_handle_vblank(dev, pipe))
@@ -3376,7 +3586,7 @@ static bool i915_handle_vblank(struct drm_device *dev,
 static irqreturn_t i915_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = (struct drm_device *) arg;
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
         unsigned long irqflags;
         u32 flip_mask =
@@ -3384,8 +3594,6 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
         int pipe, ret = IRQ_NONE;
 
-        atomic_inc(&dev_priv->irq_received);
-
         iir = I915_READ(IIR);
         do {
                 bool irq_received = (iir & ~flip_mask) != 0;
@@ -3398,7 +3606,9 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
                  */
                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
-                        i915_handle_error(dev, false);
+                        i915_handle_error(dev, false,
+                                          "Command parser error, iir 0x%08x",
+                                          iir);
 
                 for_each_pipe(pipe) {
                         int reg = PIPESTAT(pipe);
@@ -3406,9 +3616,6 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 
                         /* Clear the PIPE*STAT regs before the IIR */
                         if (pipe_stats[pipe] & 0x8000ffff) {
-                                if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
-                                        DRM_DEBUG_DRIVER("pipe %c underrun\n",
-                                                         pipe_name(pipe));
                                 I915_WRITE(reg, pipe_stats[pipe]);
                                 irq_received = true;
                         }
@@ -3424,9 +3631,6 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
                         u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
                         u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
 
-                        DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
-                                  hotplug_status);
-
                         intel_hpd_irq_handler(dev, hotplug_trigger, hpd_status_i915);
 
                         I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
@@ -3453,6 +3657,10 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 
                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
                                 i9xx_pipe_crc_irq_handler(dev, pipe);
+
+                        if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
+                            intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
+                                DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
                 }
 
                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
@@ -3484,10 +3692,10 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 
 static void i915_irq_uninstall(struct drm_device * dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
 
-        del_timer_sync(&dev_priv->hotplug_reenable_timer);
+        intel_hpd_irq_uninstall(dev_priv);
 
         if (I915_HAS_HOTPLUG(dev)) {
                 I915_WRITE(PORT_HOTPLUG_EN, 0);
@@ -3508,11 +3716,9 @@ static void i915_irq_uninstall(struct drm_device * dev)
 
 static void i965_irq_preinstall(struct drm_device * dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
 
-        atomic_set(&dev_priv->irq_received, 0);
-
         I915_WRITE(PORT_HOTPLUG_EN, 0);
         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
 
@@ -3526,7 +3732,7 @@ static void i965_irq_preinstall(struct drm_device * dev)
 
 static int i965_irq_postinstall(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 enable_mask;
         u32 error_mask;
         unsigned long irqflags;
@@ -3551,9 +3757,9 @@ static int i965_irq_postinstall(struct drm_device *dev)
         /* Interrupt setup is already guaranteed to be single-threaded, this is
          * just to make the assert_spin_locked check happy. */
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
-        i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_EVENT_ENABLE);
-        i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_ENABLE);
-        i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_ENABLE);
+        i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
+        i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
+        i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
         /*
@@ -3585,7 +3791,7 @@ static int i965_irq_postinstall(struct drm_device *dev)
 
 static void i915_hpd_irq_setup(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_mode_config *mode_config = &dev->mode_config;
         struct intel_encoder *intel_encoder;
         u32 hotplug_en;
@@ -3617,25 +3823,21 @@ static void i915_hpd_irq_setup(struct drm_device *dev)
 static irqreturn_t i965_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = (struct drm_device *) arg;
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 iir, new_iir;
         u32 pipe_stats[I915_MAX_PIPES];
         unsigned long irqflags;
-        int irq_received;
         int ret = IRQ_NONE, pipe;
         u32 flip_mask =
                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
                 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
 
-        atomic_inc(&dev_priv->irq_received);
-
         iir = I915_READ(IIR);
 
         for (;;) {
+                bool irq_received = (iir & ~flip_mask) != 0;
                 bool blc_event = false;
 
-                irq_received = (iir & ~flip_mask) != 0;
-
                 /* Can't rely on pipestat interrupt bit in iir as it might
                  * have been cleared after the pipestat interrupt was received.
                  * It doesn't set the bit in iir again, but it still produces
@@ -3643,7 +3845,9 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
                  */
                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
                 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
-                        i915_handle_error(dev, false);
+                        i915_handle_error(dev, false,
+                                          "Command parser error, iir 0x%08x",
+                                          iir);
 
                 for_each_pipe(pipe) {
                         int reg = PIPESTAT(pipe);
@@ -3653,11 +3857,8 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
                          * Clear the PIPE*STAT regs before the IIR
                          */
                         if (pipe_stats[pipe] & 0x8000ffff) {
-                                if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
-                                        DRM_DEBUG_DRIVER("pipe %c underrun\n",
-                                                         pipe_name(pipe));
                                 I915_WRITE(reg, pipe_stats[pipe]);
-                                irq_received = 1;
+                                irq_received = true;
                         }
                 }
                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
@@ -3674,9 +3875,6 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
                                                                   HOTPLUG_INT_STATUS_G4X :
                                                                   HOTPLUG_INT_STATUS_I915);
 
-                        DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
-                                  hotplug_status);
-
                         intel_hpd_irq_handler(dev, hotplug_trigger,
                                               IS_G4X(dev) ? hpd_status_g4x : hpd_status_i915);
 
@@ -3706,8 +3904,11 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 
                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
                                 i9xx_pipe_crc_irq_handler(dev, pipe);
-                }
 
+                        if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS &&
+                            intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
+                                DRM_ERROR("pipe %c underrun\n", pipe_name(pipe));
+                }
 
                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
                         intel_opregion_asle_intr(dev);
@@ -3740,13 +3941,13 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 
 static void i965_irq_uninstall(struct drm_device * dev)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
 
         if (!dev_priv)
                 return;
 
-        del_timer_sync(&dev_priv->hotplug_reenable_timer);
+        intel_hpd_irq_uninstall(dev_priv);
 
         I915_WRITE(PORT_HOTPLUG_EN, 0);
         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
@@ -3763,9 +3964,9 @@ static void i965_irq_uninstall(struct drm_device * dev)
         I915_WRITE(IIR, I915_READ(IIR));
 }
 
-static void i915_reenable_hotplug_timer_func(unsigned long data)
+static void intel_hpd_irq_reenable(unsigned long data)
 {
-        drm_i915_private_t *dev_priv = (drm_i915_private_t *)data;
+        struct drm_i915_private *dev_priv = (struct drm_i915_private *)data;
         struct drm_device *dev = dev_priv->dev;
         struct drm_mode_config *mode_config = &dev->mode_config;
         unsigned long irqflags;
@@ -3807,10 +4008,13 @@ void intel_irq_init(struct drm_device *dev)
         INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
         INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
 
+        /* Let's track the enabled rps events */
+        dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
+
         setup_timer(&dev_priv->gpu_error.hangcheck_timer,
                     i915_hangcheck_elapsed,
                     (unsigned long) dev);
-        setup_timer(&dev_priv->hotplug_reenable_timer, i915_reenable_hotplug_timer_func,
+        setup_timer(&dev_priv->hotplug_reenable_timer, intel_hpd_irq_reenable,
                     (unsigned long) dev_priv);
 
         pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
@@ -3906,32 +4110,32 @@ void intel_hpd_init(struct drm_device *dev)
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
-/* Disable interrupts so we can allow Package C8+. */
-void hsw_pc8_disable_interrupts(struct drm_device *dev)
+/* Disable interrupts so we can allow runtime PM. */
+void hsw_runtime_pm_disable_interrupts(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
 
-        dev_priv->pc8.regsave.deimr = I915_READ(DEIMR);
-        dev_priv->pc8.regsave.sdeimr = I915_READ(SDEIMR);
-        dev_priv->pc8.regsave.gtimr = I915_READ(GTIMR);
-        dev_priv->pc8.regsave.gtier = I915_READ(GTIER);
-        dev_priv->pc8.regsave.gen6_pmimr = I915_READ(GEN6_PMIMR);
+        dev_priv->pm.regsave.deimr = I915_READ(DEIMR);
+        dev_priv->pm.regsave.sdeimr = I915_READ(SDEIMR);
+        dev_priv->pm.regsave.gtimr = I915_READ(GTIMR);
+        dev_priv->pm.regsave.gtier = I915_READ(GTIER);
+        dev_priv->pm.regsave.gen6_pmimr = I915_READ(GEN6_PMIMR);
 
         ironlake_disable_display_irq(dev_priv, 0xffffffff);
         ibx_disable_display_interrupt(dev_priv, 0xffffffff);
         ilk_disable_gt_irq(dev_priv, 0xffffffff);
         snb_disable_pm_irq(dev_priv, 0xffffffff);
 
-        dev_priv->pc8.irqs_disabled = true;
+        dev_priv->pm.irqs_disabled = true;
 
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
 
-/* Restore interrupts so we can recover from Package C8+. */
-void hsw_pc8_restore_interrupts(struct drm_device *dev)
+/* Restore interrupts so we can recover from runtime PM. */
+void hsw_runtime_pm_restore_interrupts(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
@@ -3951,13 +4155,13 @@ void hsw_pc8_restore_interrupts(struct drm_device *dev)
         val = I915_READ(GEN6_PMIMR);
         WARN(val != 0xffffffff, "GEN6_PMIMR is 0x%08x\n", val);
 
-        dev_priv->pc8.irqs_disabled = false;
+        dev_priv->pm.irqs_disabled = false;
 
-        ironlake_enable_display_irq(dev_priv, ~dev_priv->pc8.regsave.deimr);
-        ibx_enable_display_interrupt(dev_priv, ~dev_priv->pc8.regsave.sdeimr);
-        ilk_enable_gt_irq(dev_priv, ~dev_priv->pc8.regsave.gtimr);
-        snb_enable_pm_irq(dev_priv, ~dev_priv->pc8.regsave.gen6_pmimr);
-        I915_WRITE(GTIER, dev_priv->pc8.regsave.gtier);
+        ironlake_enable_display_irq(dev_priv, ~dev_priv->pm.regsave.deimr);
+        ibx_enable_display_interrupt(dev_priv, ~dev_priv->pm.regsave.sdeimr);
+        ilk_enable_gt_irq(dev_priv, ~dev_priv->pm.regsave.gtimr);
+        snb_enable_pm_irq(dev_priv, ~dev_priv->pm.regsave.gen6_pmimr);
+        I915_WRITE(GTIER, dev_priv->pm.regsave.gtier);
 
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 }
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
new file mode 100644
index 000000000000..d1d7980f0e01
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -0,0 +1,154 @@
+/*
+ * Copyright © 2014 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ */
+
+#include "i915_drv.h"
+
+struct i915_params i915 __read_mostly = {
+        .modeset = -1,
+        .panel_ignore_lid = 1,
+        .powersave = 1,
+        .semaphores = -1,
+        .lvds_downclock = 0,
+        .lvds_channel_mode = 0,
+        .panel_use_ssc = -1,
+        .vbt_sdvo_panel_type = -1,
+        .enable_rc6 = -1,
+        .enable_fbc = -1,
+        .enable_hangcheck = true,
+        .enable_ppgtt = -1,
+        .enable_psr = 0,
+        .preliminary_hw_support = IS_ENABLED(CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT),
+        .disable_power_well = 1,
+        .enable_ips = 1,
+        .fastboot = 0,
+        .prefault_disable = 0,
+        .reset = true,
+        .invert_brightness = 0,
+        .disable_display = 0,
+        .enable_cmd_parser = 0,
+};
+
+module_param_named(modeset, i915.modeset, int, 0400);
+MODULE_PARM_DESC(modeset,
+        "Use kernel modesetting [KMS] (0=DRM_I915_KMS from .config, "
+        "1=on, -1=force vga console preference [default])");
+
+module_param_named(panel_ignore_lid, i915.panel_ignore_lid, int, 0600);
+MODULE_PARM_DESC(panel_ignore_lid,
+        "Override lid status (0=autodetect, 1=autodetect disabled [default], "
+        "-1=force lid closed, -2=force lid open)");
+
+module_param_named(powersave, i915.powersave, int, 0600);
+MODULE_PARM_DESC(powersave,
+        "Enable powersavings, fbc, downclocking, etc. (default: true)");
+
+module_param_named(semaphores, i915.semaphores, int, 0400);
+MODULE_PARM_DESC(semaphores,
+        "Use semaphores for inter-ring sync "
+        "(default: -1 (use per-chip defaults))");
+
+module_param_named(enable_rc6, i915.enable_rc6, int, 0400);
+MODULE_PARM_DESC(enable_rc6,
+        "Enable power-saving render C-state 6. "
+        "Different stages can be selected via bitmask values "
+        "(0 = disable; 1 = enable rc6; 2 = enable deep rc6; 4 = enable deepest rc6). "
+        "For example, 3 would enable rc6 and deep rc6, and 7 would enable everything. "
+        "default: -1 (use per-chip default)");
+
+module_param_named(enable_fbc, i915.enable_fbc, int, 0600);
+MODULE_PARM_DESC(enable_fbc,
+        "Enable frame buffer compression for power savings "
+        "(default: -1 (use per-chip default))");
+
+module_param_named(lvds_downclock, i915.lvds_downclock, int, 0400);
+MODULE_PARM_DESC(lvds_downclock,
+        "Use panel (LVDS/eDP) downclocking for power savings "
+        "(default: false)");
+
+module_param_named(lvds_channel_mode, i915.lvds_channel_mode, int, 0600);
+MODULE_PARM_DESC(lvds_channel_mode,
+         "Specify LVDS channel mode "
+         "(0=probe BIOS [default], 1=single-channel, 2=dual-channel)");
+
+module_param_named(lvds_use_ssc, i915.panel_use_ssc, int, 0600);
+MODULE_PARM_DESC(lvds_use_ssc,
+        "Use Spread Spectrum Clock with panels [LVDS/eDP] "
+        "(default: auto from VBT)");
+
+module_param_named(vbt_sdvo_panel_type, i915.vbt_sdvo_panel_type, int, 0600);
+MODULE_PARM_DESC(vbt_sdvo_panel_type,
+        "Override/Ignore selection of SDVO panel mode in the VBT "
+        "(-2=ignore, -1=auto [default], index in VBT BIOS table)");
+
+module_param_named(reset, i915.reset, bool, 0600);
+MODULE_PARM_DESC(reset, "Attempt GPU resets (default: true)");
+
+module_param_named(enable_hangcheck, i915.enable_hangcheck, bool, 0644);
+MODULE_PARM_DESC(enable_hangcheck,
+        "Periodically check GPU activity for detecting hangs. "
+        "WARNING: Disabling this can cause system wide hangs. "
+        "(default: true)");
+
+module_param_named(enable_ppgtt, i915.enable_ppgtt, int, 0400);
+MODULE_PARM_DESC(enable_ppgtt,
+        "Override PPGTT usage. "
+        "(-1=auto [default], 0=disabled, 1=aliasing, 2=full)");
+
+module_param_named(enable_psr, i915.enable_psr, int, 0600);
+MODULE_PARM_DESC(enable_psr, "Enable PSR (default: false)");
+
+module_param_named(preliminary_hw_support, i915.preliminary_hw_support, int, 0600);
+MODULE_PARM_DESC(preliminary_hw_support,
+        "Enable preliminary hardware support.");
+
+module_param_named(disable_power_well, i915.disable_power_well, int, 0600);
+MODULE_PARM_DESC(disable_power_well,
+        "Disable the power well when possible (default: true)");
+
+module_param_named(enable_ips, i915.enable_ips, int, 0600);
+MODULE_PARM_DESC(enable_ips, "Enable IPS (default: true)");
+
+module_param_named(fastboot, i915.fastboot, bool, 0600);
+MODULE_PARM_DESC(fastboot,
+        "Try to skip unnecessary mode sets at boot time (default: false)");
+
+module_param_named(prefault_disable, i915.prefault_disable, bool, 0600);
+MODULE_PARM_DESC(prefault_disable,
+        "Disable page prefaulting for pread/pwrite/reloc (default:false). "
+        "For developers only.");
+
+module_param_named(invert_brightness, i915.invert_brightness, int, 0600);
+MODULE_PARM_DESC(invert_brightness,
+        "Invert backlight brightness "
+        "(-1 force normal, 0 machine defaults, 1 force inversion), please "
+        "report PCI device ID, subsystem vendor and subsystem device ID "
+        "to dri-devel@lists.freedesktop.org, if your machine needs it. "
+        "It will then be included in an upcoming module version.");
+
+module_param_named(disable_display, i915.disable_display, bool, 0600);
+MODULE_PARM_DESC(disable_display, "Disable display (default: false)");
+
+module_param_named(enable_cmd_parser, i915.enable_cmd_parser, int, 0600);
+MODULE_PARM_DESC(enable_cmd_parser,
+                 "Enable command parsing (1=enabled, 0=disabled [default])");
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index a48b7cad6f11..9f5b18d9d885 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -26,7 +26,6 @@
 #define _I915_REG_H_
 
 #define _PIPE(pipe, a, b) ((a) + (pipe)*((b)-(a)))
-#define _PIPE_INC(pipe, base, inc) ((base) + (pipe)*(inc))
 #define _TRANSCODER(tran, a, b) ((a) + (tran)*((b)-(a)))
 
 #define _PORT(port, a, b) ((a) + (port)*((b)-(a)))
@@ -73,7 +72,8 @@
 #define   I915_GC_RENDER_CLOCK_166_MHZ  (0 << 0)
 #define   I915_GC_RENDER_CLOCK_200_MHZ  (1 << 0)
 #define   I915_GC_RENDER_CLOCK_333_MHZ  (4 << 0)
-#define LBB     0xf4
+#define PCI_LBPC 0xf4 /* legacy/combination backlight modes, also called LBB */
+
 
 /* Graphics reset regs */
 #define I965_GDRST 0xc0 /* PCI config register */
@@ -175,6 +175,18 @@
 #define VGA_CR_DATA_CGA 0x3d5
 
 /*
+ * Instruction field definitions used by the command parser
+ */
+#define INSTR_CLIENT_SHIFT      29
+#define INSTR_CLIENT_MASK       0xE0000000
+#define   INSTR_MI_CLIENT       0x0
+#define   INSTR_BC_CLIENT       0x2
+#define   INSTR_RC_CLIENT       0x3
+#define INSTR_SUBCLIENT_SHIFT   27
+#define INSTR_SUBCLIENT_MASK    0x18000000
+#define   INSTR_MEDIA_SUBCLIENT 0x2
+
+/*
  * Memory interface instructions used by the kernel
  */
 #define MI_INSTR(opcode, flags) (((opcode) << 23) | (flags))
@@ -377,14 +389,30 @@
 #define   DSPFREQSTAT_MASK                      (0x3 << DSPFREQSTAT_SHIFT)
 #define   DSPFREQGUAR_SHIFT                     14
 #define   DSPFREQGUAR_MASK                      (0x3 << DSPFREQGUAR_SHIFT)
+
+/* See the PUNIT HAS v0.8 for the below bits */
+enum punit_power_well {
+        PUNIT_POWER_WELL_RENDER                 = 0,
+        PUNIT_POWER_WELL_MEDIA                  = 1,
+        PUNIT_POWER_WELL_DISP2D                 = 3,
+        PUNIT_POWER_WELL_DPIO_CMN_BC            = 5,
+        PUNIT_POWER_WELL_DPIO_TX_B_LANES_01     = 6,
+        PUNIT_POWER_WELL_DPIO_TX_B_LANES_23     = 7,
+        PUNIT_POWER_WELL_DPIO_TX_C_LANES_01     = 8,
+        PUNIT_POWER_WELL_DPIO_TX_C_LANES_23     = 9,
+        PUNIT_POWER_WELL_DPIO_RX0               = 10,
+        PUNIT_POWER_WELL_DPIO_RX1               = 11,
+
+        PUNIT_POWER_WELL_NUM,
+};
+
 #define PUNIT_REG_PWRGT_CTRL                    0x60
 #define PUNIT_REG_PWRGT_STATUS                  0x61
-#define   PUNIT_CLK_GATE                        1
-#define   PUNIT_PWR_RESET                       2
-#define   PUNIT_PWR_GATE                        3
-#define   RENDER_PWRGT                          (PUNIT_PWR_GATE << 0)
-#define   MEDIA_PWRGT                           (PUNIT_PWR_GATE << 2)
-#define   DISP2D_PWRGT                          (PUNIT_PWR_GATE << 6)
+#define   PUNIT_PWRGT_MASK(power_well)          (3 << ((power_well) * 2))
+#define   PUNIT_PWRGT_PWR_ON(power_well)        (0 << ((power_well) * 2))
+#define   PUNIT_PWRGT_CLK_GATE(power_well)      (1 << ((power_well) * 2))
+#define   PUNIT_PWRGT_RESET(power_well)         (2 << ((power_well) * 2))
+#define   PUNIT_PWRGT_PWR_GATE(power_well)      (3 << ((power_well) * 2))
 
 #define PUNIT_REG_GPU_LFM                       0xd3
 #define PUNIT_REG_GPU_FREQ_REQ                  0xd4
@@ -678,6 +706,7 @@
 #define BLT_HWS_PGA_GEN7        (0x04280)
 #define VEBOX_HWS_PGA_GEN7      (0x04380)
 #define RING_ACTHD(base)        ((base)+0x74)
+#define RING_ACTHD_UDW(base)    ((base)+0x5c)
 #define RING_NOPID(base)        ((base)+0x94)
 #define RING_IMR(base)          ((base)+0xa8)
 #define RING_TIMESTAMP(base)    ((base)+0x358)
@@ -720,6 +749,7 @@
 #define RING_INSTPS(base)       ((base)+0x70)
 #define RING_DMA_FADD(base)     ((base)+0x78)
 #define RING_INSTPM(base)       ((base)+0xc0)
+#define RING_MI_MODE(base)      ((base)+0x9c)
 #define INSTPS          0x02070 /* 965+ only */
 #define INSTDONE1       0x0207c /* 965+ only */
 #define ACTHD_I965      0x02074
@@ -789,15 +819,22 @@
 #define _3D_CHICKEN3    0x02090
 #define  _3D_CHICKEN_SF_DISABLE_OBJEND_CULL             (1 << 10)
 #define  _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL          (1 << 5)
-#define  _3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(x)       ((x)<<1)
+#define  _3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(x)       ((x)<<1) /* gen8+ */
+#define  _3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH   (1 << 1) /* gen6 */
 
 #define MI_MODE         0x0209c
 # define VS_TIMER_DISPATCH                              (1 << 6)
 # define MI_FLUSH_ENABLE                                (1 << 12)
 # define ASYNC_FLIP_PERF_DISABLE                        (1 << 14)
+# define MODE_IDLE                                      (1 << 9)
 
 #define GEN6_GT_MODE    0x20d0
-#define   GEN6_GT_MODE_HI                               (1 << 9)
+#define GEN7_GT_MODE    0x7008
+#define   GEN6_WIZ_HASHING(hi, lo)                      (((hi) << 9) | ((lo) << 7))
+#define   GEN6_WIZ_HASHING_8x8                          GEN6_WIZ_HASHING(0, 0)
+#define   GEN6_WIZ_HASHING_8x4                          GEN6_WIZ_HASHING(0, 1)
+#define   GEN6_WIZ_HASHING_16x4                         GEN6_WIZ_HASHING(1, 0)
+#define   GEN6_WIZ_HASHING_MASK                         (GEN6_WIZ_HASHING(1, 1) << 16)
 #define   GEN6_TD_FOUR_ROW_DISPATCH_DISABLE             (1 << 5)
 
 #define GFX_MODE        0x02520
@@ -934,13 +971,19 @@
 #define   ECO_GATING_CX_ONLY    (1<<3)
 #define   ECO_FLIP_DONE         (1<<0)
 
+#define CACHE_MODE_0_GEN7       0x7000 /* IVB+ */
+#define   HIZ_RAW_STALL_OPT_DISABLE (1<<2)
 #define CACHE_MODE_1            0x7004 /* IVB+ */
-#define   PIXEL_SUBSPAN_COLLECT_OPT_DISABLE (1<<6)
+#define   PIXEL_SUBSPAN_COLLECT_OPT_DISABLE     (1<<6)
+#define   GEN8_4x4_STC_OPTIMIZATION_DISABLE     (1<<6)
 
 #define GEN6_BLITTER_ECOSKPD    0x221d0
 #define   GEN6_BLITTER_LOCK_SHIFT                       16
 #define   GEN6_BLITTER_FBC_NOTIFY                       (1<<3)
 
+#define GEN6_RC_SLEEP_PSMI_CONTROL      0x2050
+#define   GEN8_RC_SEMA_IDLE_MSG_DISABLE (1 << 12)
+
 #define GEN6_BSD_SLEEP_PSMI_CONTROL     0x12050
 #define   GEN6_BSD_SLEEP_MSG_DISABLE    (1 << 0)
 #define   GEN6_BSD_SLEEP_FLUSH_DISABLE  (1 << 2)
@@ -1046,9 +1089,8 @@
 #define   FBC_CTL_IDLE_LINE     (2<<2)
 #define   FBC_CTL_IDLE_DEBUG    (3<<2)
 #define   FBC_CTL_CPU_FENCE     (1<<1)
-#define   FBC_CTL_PLANEA        (0<<0)
-#define   FBC_CTL_PLANEB        (1<<0)
-#define FBC_FENCE_OFF           0x0321b
+#define   FBC_CTL_PLANE(plane)  ((plane)<<0)
+#define FBC_FENCE_OFF           0x03218 /* BSpec typo has 321Bh */
 #define FBC_TAG                 0x03300
 
 #define FBC_LL_SIZE             (1536)
@@ -1057,9 +1099,8 @@
 #define DPFC_CB_BASE            0x3200
 #define DPFC_CONTROL            0x3208
 #define   DPFC_CTL_EN           (1<<31)
-#define   DPFC_CTL_PLANEA       (0<<30)
-#define   DPFC_CTL_PLANEB       (1<<30)
-#define   IVB_DPFC_CTL_PLANE_SHIFT      (29)
+#define   DPFC_CTL_PLANE(plane) ((plane)<<30)
+#define   IVB_DPFC_CTL_PLANE(plane)     ((plane)<<29)
 #define   DPFC_CTL_FENCE_EN     (1<<29)
 #define   IVB_DPFC_CTL_FENCE_EN (1<<28)
 #define   DPFC_CTL_PERSISTENT_MODE      (1<<25)
@@ -1120,13 +1161,6 @@
 #define   FBC_REND_NUKE         (1<<2)
 #define   FBC_REND_CACHE_CLEAN  (1<<1)
 
-#define _HSW_PIPE_SLICE_CHICKEN_1_A     0x420B0
-#define _HSW_PIPE_SLICE_CHICKEN_1_B     0x420B4
-#define   HSW_BYPASS_FBC_QUEUE          (1<<22)
-#define HSW_PIPE_SLICE_CHICKEN_1(pipe) _PIPE(pipe, + \
-                                             _HSW_PIPE_SLICE_CHICKEN_1_A, + \
-                                             _HSW_PIPE_SLICE_CHICKEN_1_B)
-
 /*
  * GPIO regs
  */
@@ -1202,6 +1236,10 @@
 /*
  * Clock control & power management
  */
+#define DPLL_A_OFFSET 0x6014
+#define DPLL_B_OFFSET 0x6018
+#define DPLL(pipe) (dev_priv->info.dpll_offsets[pipe] + \
+                    dev_priv->info.display_mmio_offset)
 
 #define VGA0    0x6000
 #define VGA1    0x6004
@@ -1214,9 +1252,6 @@
 #define   VGA1_PD_P1_DIV_2      (1 << 13)
 #define   VGA1_PD_P1_SHIFT      8
 #define   VGA1_PD_P1_MASK       (0x1f << 8)
-#define _DPLL_A (dev_priv->info->display_mmio_offset + 0x6014)
-#define _DPLL_B (dev_priv->info->display_mmio_offset + 0x6018)
-#define DPLL(pipe) _PIPE(pipe, _DPLL_A, _DPLL_B)
 #define   DPLL_VCO_ENABLE               (1 << 31)
 #define   DPLL_SDVO_HIGH_SPEED          (1 << 30)
 #define   DPLL_DVO_2X_MODE              (1 << 30)
@@ -1278,7 +1313,12 @@
 #define   SDVO_MULTIPLIER_MASK                  0x000000ff
 #define   SDVO_MULTIPLIER_SHIFT_HIRES           4
 #define   SDVO_MULTIPLIER_SHIFT_VGA             0
-#define _DPLL_A_MD (dev_priv->info->display_mmio_offset + 0x601c) /* 965+ only */
+
+#define DPLL_A_MD_OFFSET 0x601c /* 965+ only */
+#define DPLL_B_MD_OFFSET 0x6020 /* 965+ only */
+#define DPLL_MD(pipe) (dev_priv->info.dpll_md_offsets[pipe] + \
+                       dev_priv->info.display_mmio_offset)
+
 /*
  * UDI pixel divider, controlling how many pixels are stuffed into a packet.
  *
@@ -1315,8 +1355,6 @@
  */
 #define   DPLL_MD_VGA_UDI_MULTIPLIER_MASK       0x0000003f
 #define   DPLL_MD_VGA_UDI_MULTIPLIER_SHIFT      0
-#define _DPLL_B_MD (dev_priv->info->display_mmio_offset + 0x6020) /* 965+ only */
-#define DPLL_MD(pipe) _PIPE(pipe, _DPLL_A_MD, _DPLL_B_MD)
 
 #define _FPA0   0x06040
 #define _FPA1   0x06044
@@ -1348,7 +1386,7 @@
 #define  DSTATE_PLL_D3_OFF                      (1<<3)
 #define  DSTATE_GFX_CLOCK_GATING                (1<<1)
 #define  DSTATE_DOT_CLOCK_GATING                (1<<0)
-#define DSPCLK_GATE_D   (dev_priv->info->display_mmio_offset + 0x6200)
+#define DSPCLK_GATE_D   (dev_priv->info.display_mmio_offset + 0x6200)
 # define DPUNIT_B_CLOCK_GATE_DISABLE            (1 << 30) /* 965 */
 # define VSUNIT_CLOCK_GATE_DISABLE              (1 << 29) /* 965 */
 # define VRHUNIT_CLOCK_GATE_DISABLE             (1 << 28) /* 965 */
@@ -1472,10 +1510,10 @@
 /*
  * Palette regs
  */
-
-#define _PALETTE_A              (dev_priv->info->display_mmio_offset + 0xa000)
-#define _PALETTE_B              (dev_priv->info->display_mmio_offset + 0xa800)
-#define PALETTE(pipe) _PIPE(pipe, _PALETTE_A, _PALETTE_B)
+#define PALETTE_A_OFFSET 0xa000
+#define PALETTE_B_OFFSET 0xa800
+#define PALETTE(pipe) (dev_priv->info.palette_offsets[pipe] + \
+                       dev_priv->info.display_mmio_offset)
 
 /* MCH MMIO space */
 
@@ -1862,7 +1900,7 @@
  */
 
 /* Pipe A CRC regs */
-#define _PIPE_CRC_CTL_A         (dev_priv->info->display_mmio_offset + 0x60050)
+#define _PIPE_CRC_CTL_A                 0x60050
 #define   PIPE_CRC_ENABLE               (1 << 31)
 /* ivb+ source selection */
 #define   PIPE_CRC_SOURCE_PRIMARY_IVB   (0 << 29)
@@ -1902,11 +1940,11 @@
 #define _PIPE_CRC_RES_4_A_IVB           0x60070
 #define _PIPE_CRC_RES_5_A_IVB           0x60074
 
-#define _PIPE_CRC_RES_RED_A             (dev_priv->info->display_mmio_offset + 0x60060)
-#define _PIPE_CRC_RES_GREEN_A           (dev_priv->info->display_mmio_offset + 0x60064)
-#define _PIPE_CRC_RES_BLUE_A            (dev_priv->info->display_mmio_offset + 0x60068)
-#define _PIPE_CRC_RES_RES1_A_I915       (dev_priv->info->display_mmio_offset + 0x6006c)
-#define _PIPE_CRC_RES_RES2_A_G4X        (dev_priv->info->display_mmio_offset + 0x60080)
+#define _PIPE_CRC_RES_RED_A             0x60060
+#define _PIPE_CRC_RES_GREEN_A           0x60064
+#define _PIPE_CRC_RES_BLUE_A            0x60068
+#define _PIPE_CRC_RES_RES1_A_I915       0x6006c
+#define _PIPE_CRC_RES_RES2_A_G4X        0x60080
 
 /* Pipe B CRC regs */
 #define _PIPE_CRC_RES_1_B_IVB           0x61064
@@ -1915,59 +1953,69 @@
 #define _PIPE_CRC_RES_4_B_IVB           0x61070
 #define _PIPE_CRC_RES_5_B_IVB           0x61074
 
-#define PIPE_CRC_CTL(pipe)      _PIPE_INC(pipe, _PIPE_CRC_CTL_A, 0x01000)
+#define PIPE_CRC_CTL(pipe) _TRANSCODER2(pipe, _PIPE_CRC_CTL_A)
 #define PIPE_CRC_RES_1_IVB(pipe)        \
-        _PIPE(pipe, _PIPE_CRC_RES_1_A_IVB, _PIPE_CRC_RES_1_B_IVB)
+        _TRANSCODER2(pipe, _PIPE_CRC_RES_1_A_IVB)
 #define PIPE_CRC_RES_2_IVB(pipe)        \
-        _PIPE(pipe, _PIPE_CRC_RES_2_A_IVB, _PIPE_CRC_RES_2_B_IVB)
+        _TRANSCODER2(pipe, _PIPE_CRC_RES_2_A_IVB)
 #define PIPE_CRC_RES_3_IVB(pipe)        \
-        _PIPE(pipe, _PIPE_CRC_RES_3_A_IVB, _PIPE_CRC_RES_3_B_IVB)
+        _TRANSCODER2(pipe, _PIPE_CRC_RES_3_A_IVB)
 #define PIPE_CRC_RES_4_IVB(pipe)        \
-        _PIPE(pipe, _PIPE_CRC_RES_4_A_IVB, _PIPE_CRC_RES_4_B_IVB)
+        _TRANSCODER2(pipe, _PIPE_CRC_RES_4_A_IVB)
 #define PIPE_CRC_RES_5_IVB(pipe)        \
-        _PIPE(pipe, _PIPE_CRC_RES_5_A_IVB, _PIPE_CRC_RES_5_B_IVB)
+        _TRANSCODER2(pipe, _PIPE_CRC_RES_5_A_IVB)
 
 #define PIPE_CRC_RES_RED(pipe) \
-        _PIPE_INC(pipe, _PIPE_CRC_RES_RED_A, 0x01000)
+        _TRANSCODER2(pipe, _PIPE_CRC_RES_RED_A)
 #define PIPE_CRC_RES_GREEN(pipe) \
-        _PIPE_INC(pipe, _PIPE_CRC_RES_GREEN_A, 0x01000)
+        _TRANSCODER2(pipe, _PIPE_CRC_RES_GREEN_A)
 #define PIPE_CRC_RES_BLUE(pipe) \
-        _PIPE_INC(pipe, _PIPE_CRC_RES_BLUE_A, 0x01000)
+        _TRANSCODER2(pipe, _PIPE_CRC_RES_BLUE_A)
 #define PIPE_CRC_RES_RES1_I915(pipe) \
-        _PIPE_INC(pipe, _PIPE_CRC_RES_RES1_A_I915, 0x01000)
+        _TRANSCODER2(pipe, _PIPE_CRC_RES_RES1_A_I915)
 #define PIPE_CRC_RES_RES2_G4X(pipe) \
-        _PIPE_INC(pipe, _PIPE_CRC_RES_RES2_A_G4X, 0x01000)
+        _TRANSCODER2(pipe, _PIPE_CRC_RES_RES2_A_G4X)
 
 /* Pipe A timing regs */
-#define _HTOTAL_A       (dev_priv->info->display_mmio_offset + 0x60000)
-#define _HBLANK_A       (dev_priv->info->display_mmio_offset + 0x60004)
-#define _HSYNC_A        (dev_priv->info->display_mmio_offset + 0x60008)
-#define _VTOTAL_A       (dev_priv->info->display_mmio_offset + 0x6000c)
-#define _VBLANK_A       (dev_priv->info->display_mmio_offset + 0x60010)
-#define _VSYNC_A        (dev_priv->info->display_mmio_offset + 0x60014)
-#define _PIPEASRC       (dev_priv->info->display_mmio_offset + 0x6001c)
-#define _BCLRPAT_A      (dev_priv->info->display_mmio_offset + 0x60020)
-#define _VSYNCSHIFT_A   (dev_priv->info->display_mmio_offset + 0x60028)
+#define _HTOTAL_A       0x60000
+#define _HBLANK_A       0x60004
+#define _HSYNC_A        0x60008
+#define _VTOTAL_A       0x6000c
+#define _VBLANK_A       0x60010
+#define _VSYNC_A        0x60014
+#define _PIPEASRC       0x6001c
+#define _BCLRPAT_A      0x60020
+#define _VSYNCSHIFT_A   0x60028
 
 /* Pipe B timing regs */
-#define _HTOTAL_B       (dev_priv->info->display_mmio_offset + 0x61000)
-#define _HBLANK_B       (dev_priv->info->display_mmio_offset + 0x61004)
-#define _HSYNC_B        (dev_priv->info->display_mmio_offset + 0x61008)
-#define _VTOTAL_B       (dev_priv->info->display_mmio_offset + 0x6100c)
-#define _VBLANK_B       (dev_priv->info->display_mmio_offset + 0x61010)
-#define _VSYNC_B        (dev_priv->info->display_mmio_offset + 0x61014)
-#define _PIPEBSRC       (dev_priv->info->display_mmio_offset + 0x6101c)
-#define _BCLRPAT_B      (dev_priv->info->display_mmio_offset + 0x61020)
-#define _VSYNCSHIFT_B   (dev_priv->info->display_mmio_offset + 0x61028)
-
-#define HTOTAL(trans) _TRANSCODER(trans, _HTOTAL_A, _HTOTAL_B)
-#define HBLANK(trans) _TRANSCODER(trans, _HBLANK_A, _HBLANK_B)
-#define HSYNC(trans) _TRANSCODER(trans, _HSYNC_A, _HSYNC_B)
-#define VTOTAL(trans) _TRANSCODER(trans, _VTOTAL_A, _VTOTAL_B)
-#define VBLANK(trans) _TRANSCODER(trans, _VBLANK_A, _VBLANK_B)
-#define VSYNC(trans) _TRANSCODER(trans, _VSYNC_A, _VSYNC_B)
-#define BCLRPAT(pipe) _PIPE(pipe, _BCLRPAT_A, _BCLRPAT_B)
-#define VSYNCSHIFT(trans) _TRANSCODER(trans, _VSYNCSHIFT_A, _VSYNCSHIFT_B)
+#define _HTOTAL_B       0x61000
+#define _HBLANK_B       0x61004
+#define _HSYNC_B        0x61008
+#define _VTOTAL_B       0x6100c
+#define _VBLANK_B       0x61010
+#define _VSYNC_B        0x61014
+#define _PIPEBSRC       0x6101c
+#define _BCLRPAT_B      0x61020
+#define _VSYNCSHIFT_B   0x61028
+
+#define TRANSCODER_A_OFFSET 0x60000
+#define TRANSCODER_B_OFFSET 0x61000
+#define TRANSCODER_C_OFFSET 0x62000
+#define TRANSCODER_EDP_OFFSET 0x6f000
+
+#define _TRANSCODER2(pipe, reg) (dev_priv->info.trans_offsets[(pipe)] - \
+        dev_priv->info.trans_offsets[TRANSCODER_A] + (reg) + \
+        dev_priv->info.display_mmio_offset)
+
+#define HTOTAL(trans) _TRANSCODER2(trans, _HTOTAL_A)
+#define HBLANK(trans) _TRANSCODER2(trans, _HBLANK_A)
+#define HSYNC(trans) _TRANSCODER2(trans, _HSYNC_A)
+#define VTOTAL(trans) _TRANSCODER2(trans, _VTOTAL_A)
+#define VBLANK(trans) _TRANSCODER2(trans, _VBLANK_A)
+#define VSYNC(trans) _TRANSCODER2(trans, _VSYNC_A)
+#define BCLRPAT(trans) _TRANSCODER2(trans, _BCLRPAT_A)
+#define VSYNCSHIFT(trans) _TRANSCODER2(trans, _VSYNCSHIFT_A)
+#define PIPESRC(trans) _TRANSCODER2(trans, _PIPEASRC)
 
 /* HSW+ eDP PSR registers */
 #define EDP_PSR_BASE(dev)                       (IS_HASWELL(dev) ? 0x64800 : 0x6f800)
@@ -2084,7 +2132,7 @@
 
 
 /* Hotplug control (945+ only) */
-#define PORT_HOTPLUG_EN         (dev_priv->info->display_mmio_offset + 0x61110)
+#define PORT_HOTPLUG_EN         (dev_priv->info.display_mmio_offset + 0x61110)
 #define   PORTB_HOTPLUG_INT_EN                  (1 << 29)
 #define   PORTC_HOTPLUG_INT_EN                  (1 << 28)
 #define   PORTD_HOTPLUG_INT_EN                  (1 << 27)
@@ -2114,7 +2162,7 @@
 #define CRT_HOTPLUG_DETECT_VOLTAGE_325MV        (0 << 2)
 #define CRT_HOTPLUG_DETECT_VOLTAGE_475MV        (1 << 2)
 
-#define PORT_HOTPLUG_STAT       (dev_priv->info->display_mmio_offset + 0x61114)
+#define PORT_HOTPLUG_STAT       (dev_priv->info.display_mmio_offset + 0x61114)
 /*
  * HDMI/DP bits are gen4+
  *
@@ -2332,9 +2380,7 @@
 #define VIDEO_DIP_CTL           0x61170
 /* Pre HSW: */
 #define   VIDEO_DIP_ENABLE              (1 << 31)
-#define   VIDEO_DIP_PORT_B              (1 << 29)
-#define   VIDEO_DIP_PORT_C              (2 << 29)
-#define   VIDEO_DIP_PORT_D              (3 << 29)
+#define   VIDEO_DIP_PORT(port)          ((port) << 29)
 #define   VIDEO_DIP_PORT_MASK           (3 << 29)
 #define   VIDEO_DIP_ENABLE_GCP          (1 << 25)
 #define   VIDEO_DIP_ENABLE_AVI          (1 << 21)
@@ -2391,7 +2437,7 @@
 #define PP_DIVISOR      0x61210
 
 /* Panel fitting */
-#define PFIT_CONTROL    (dev_priv->info->display_mmio_offset + 0x61230)
+#define PFIT_CONTROL    (dev_priv->info.display_mmio_offset + 0x61230)
 #define   PFIT_ENABLE           (1 << 31)
 #define   PFIT_PIPE_MASK        (3 << 29)
 #define   PFIT_PIPE_SHIFT       29
@@ -2409,7 +2455,7 @@
 #define   PFIT_SCALING_PROGRAMMED (1 << 26)
 #define   PFIT_SCALING_PILLAR   (2 << 26)
 #define   PFIT_SCALING_LETTER   (3 << 26)
-#define PFIT_PGM_RATIOS (dev_priv->info->display_mmio_offset + 0x61234)
+#define PFIT_PGM_RATIOS (dev_priv->info.display_mmio_offset + 0x61234)
 /* Pre-965 */
 #define         PFIT_VERT_SCALE_SHIFT           20
 #define         PFIT_VERT_SCALE_MASK            0xfff00000
@@ -2421,25 +2467,25 @@
 #define         PFIT_HORIZ_SCALE_SHIFT_965      0
 #define         PFIT_HORIZ_SCALE_MASK_965       0x00001fff
 
-#define PFIT_AUTO_RATIOS (dev_priv->info->display_mmio_offset + 0x61238)
+#define PFIT_AUTO_RATIOS (dev_priv->info.display_mmio_offset + 0x61238)
 
-#define _VLV_BLC_PWM_CTL2_A (dev_priv->info->display_mmio_offset + 0x61250)
-#define _VLV_BLC_PWM_CTL2_B (dev_priv->info->display_mmio_offset + 0x61350)
+#define _VLV_BLC_PWM_CTL2_A (dev_priv->info.display_mmio_offset + 0x61250)
+#define _VLV_BLC_PWM_CTL2_B (dev_priv->info.display_mmio_offset + 0x61350)
 #define VLV_BLC_PWM_CTL2(pipe) _PIPE(pipe, _VLV_BLC_PWM_CTL2_A, \
                                      _VLV_BLC_PWM_CTL2_B)
 
-#define _VLV_BLC_PWM_CTL_A (dev_priv->info->display_mmio_offset + 0x61254)
-#define _VLV_BLC_PWM_CTL_B (dev_priv->info->display_mmio_offset + 0x61354)
+#define _VLV_BLC_PWM_CTL_A (dev_priv->info.display_mmio_offset + 0x61254)
+#define _VLV_BLC_PWM_CTL_B (dev_priv->info.display_mmio_offset + 0x61354)
 #define VLV_BLC_PWM_CTL(pipe) _PIPE(pipe, _VLV_BLC_PWM_CTL_A, \
                                     _VLV_BLC_PWM_CTL_B)
 
-#define _VLV_BLC_HIST_CTL_A (dev_priv->info->display_mmio_offset + 0x61260)
-#define _VLV_BLC_HIST_CTL_B (dev_priv->info->display_mmio_offset + 0x61360)
+#define _VLV_BLC_HIST_CTL_A (dev_priv->info.display_mmio_offset + 0x61260)
+#define _VLV_BLC_HIST_CTL_B (dev_priv->info.display_mmio_offset + 0x61360)
 #define VLV_BLC_HIST_CTL(pipe) _PIPE(pipe, _VLV_BLC_HIST_CTL_A, \
                                      _VLV_BLC_HIST_CTL_B)
 
 /* Backlight control */
-#define BLC_PWM_CTL2    (dev_priv->info->display_mmio_offset + 0x61250) /* 965+ only */
+#define BLC_PWM_CTL2    (dev_priv->info.display_mmio_offset + 0x61250) /* 965+ only */
 #define   BLM_PWM_ENABLE                (1 << 31)
 #define   BLM_COMBINATION_MODE          (1 << 30) /* gen4 only */
 #define   BLM_PIPE_SELECT               (1 << 29)
@@ -2462,7 +2508,7 @@
 #define   BLM_PHASE_IN_COUNT_MASK       (0xff << 8)
 #define   BLM_PHASE_IN_INCR_SHIFT       (0)
 #define   BLM_PHASE_IN_INCR_MASK        (0xff << 0)
-#define BLC_PWM_CTL     (dev_priv->info->display_mmio_offset + 0x61254)
+#define BLC_PWM_CTL     (dev_priv->info.display_mmio_offset + 0x61254)
 /*
  * This is the most significant 15 bits of the number of backlight cycles in a
  * complete cycle of the modulated backlight control.
@@ -2484,7 +2530,7 @@
 #define   BACKLIGHT_DUTY_CYCLE_MASK_PNV         (0xfffe)
 #define   BLM_POLARITY_PNV                      (1 << 0) /* pnv only */
 
-#define BLC_HIST_CTL    (dev_priv->info->display_mmio_offset + 0x61260)
+#define BLC_HIST_CTL    (dev_priv->info.display_mmio_offset + 0x61260)
 
 /* New registers for PCH-split platforms. Safe where new bits show up, the
  * register layout machtes with gen4 BLC_PWM_CTL[12]. */
@@ -3178,10 +3224,10 @@
 /* Display & cursor control */
 
 /* Pipe A */
-#define _PIPEADSL               (dev_priv->info->display_mmio_offset + 0x70000)
+#define _PIPEADSL               0x70000
 #define   DSL_LINEMASK_GEN2     0x00000fff
 #define   DSL_LINEMASK_GEN3     0x00001fff
-#define _PIPEACONF              (dev_priv->info->display_mmio_offset + 0x70008)
+#define _PIPEACONF              0x70008
 #define   PIPECONF_ENABLE       (1<<31)
 #define   PIPECONF_DISABLE      0
 #define   PIPECONF_DOUBLE_WIDE  (1<<30)
@@ -3224,9 +3270,9 @@
 #define   PIPECONF_DITHER_TYPE_ST1 (1<<2)
 #define   PIPECONF_DITHER_TYPE_ST2 (2<<2)
 #define   PIPECONF_DITHER_TYPE_TEMP (3<<2)
-#define _PIPEASTAT              (dev_priv->info->display_mmio_offset + 0x70024)
+#define _PIPEASTAT              0x70024
 #define   PIPE_FIFO_UNDERRUN_STATUS             (1UL<<31)
-#define   SPRITE1_FLIPDONE_INT_EN_VLV           (1UL<<30)
+#define   SPRITE1_FLIP_DONE_INT_EN_VLV          (1UL<<30)
 #define   PIPE_CRC_ERROR_ENABLE                 (1UL<<29)
 #define   PIPE_CRC_DONE_ENABLE                  (1UL<<28)
 #define   PIPE_GMBUS_EVENT_ENABLE               (1UL<<27)
@@ -3239,35 +3285,55 @@
 #define   PIPE_LEGACY_BLC_EVENT_ENABLE          (1UL<<22)
 #define   PIPE_ODD_FIELD_INTERRUPT_ENABLE       (1UL<<21)
 #define   PIPE_EVEN_FIELD_INTERRUPT_ENABLE      (1UL<<20)
+#define   PIPE_B_PSR_INTERRUPT_ENABLE_VLV       (1UL<<19)
 #define   PIPE_HOTPLUG_TV_INTERRUPT_ENABLE      (1UL<<18) /* pre-965 */
 #define   PIPE_START_VBLANK_INTERRUPT_ENABLE    (1UL<<18) /* 965 or later */
 #define   PIPE_VBLANK_INTERRUPT_ENABLE          (1UL<<17)
 #define   PIPEA_HBLANK_INT_EN_VLV               (1UL<<16)
 #define   PIPE_OVERLAY_UPDATED_ENABLE           (1UL<<16)
-#define   SPRITE1_FLIPDONE_INT_STATUS_VLV       (1UL<<15)
-#define   SPRITE0_FLIPDONE_INT_STATUS_VLV       (1UL<<14)
+#define   SPRITE1_FLIP_DONE_INT_STATUS_VLV      (1UL<<15)
+#define   SPRITE0_FLIP_DONE_INT_STATUS_VLV      (1UL<<14)
 #define   PIPE_CRC_ERROR_INTERRUPT_STATUS       (1UL<<13)
 #define   PIPE_CRC_DONE_INTERRUPT_STATUS        (1UL<<12)
 #define   PIPE_GMBUS_INTERRUPT_STATUS           (1UL<<11)
-#define   PLANE_FLIPDONE_INT_STATUS_VLV         (1UL<<10)
+#define   PLANE_FLIP_DONE_INT_STATUS_VLV        (1UL<<10)
 #define   PIPE_HOTPLUG_INTERRUPT_STATUS         (1UL<<10)
 #define   PIPE_VSYNC_INTERRUPT_STATUS           (1UL<<9)
 #define   PIPE_DISPLAY_LINE_COMPARE_STATUS      (1UL<<8)
 #define   PIPE_DPST_EVENT_STATUS                (1UL<<7)
 #define   PIPE_LEGACY_BLC_EVENT_STATUS          (1UL<<6)
+#define   PIPE_A_PSR_STATUS_VLV                 (1UL<<6)
 #define   PIPE_ODD_FIELD_INTERRUPT_STATUS       (1UL<<5)
 #define   PIPE_EVEN_FIELD_INTERRUPT_STATUS      (1UL<<4)
+#define   PIPE_B_PSR_STATUS_VLV                 (1UL<<3)
 #define   PIPE_HOTPLUG_TV_INTERRUPT_STATUS      (1UL<<2) /* pre-965 */
 #define   PIPE_START_VBLANK_INTERRUPT_STATUS    (1UL<<2) /* 965 or later */
 #define   PIPE_VBLANK_INTERRUPT_STATUS          (1UL<<1)
 #define   PIPE_OVERLAY_UPDATED_STATUS           (1UL<<0)
 
-#define PIPESRC(pipe) _PIPE(pipe, _PIPEASRC, _PIPEBSRC)
-#define PIPECONF(tran) _TRANSCODER(tran, _PIPEACONF, _PIPEBCONF)
-#define PIPEDSL(pipe)  _PIPE(pipe, _PIPEADSL, _PIPEBDSL)
-#define PIPEFRAME(pipe) _PIPE(pipe, _PIPEAFRAMEHIGH, _PIPEBFRAMEHIGH)
-#define PIPEFRAMEPIXEL(pipe)  _PIPE(pipe, _PIPEAFRAMEPIXEL, _PIPEBFRAMEPIXEL)
-#define PIPESTAT(pipe) _PIPE(pipe, _PIPEASTAT, _PIPEBSTAT)
+#define PIPESTAT_INT_ENABLE_MASK                0x7fff0000
+#define PIPESTAT_INT_STATUS_MASK                0x0000ffff
+
+#define PIPE_A_OFFSET   0x70000
+#define PIPE_B_OFFSET   0x71000
+#define PIPE_C_OFFSET   0x72000
+/*
+ * There's actually no pipe EDP. Some pipe registers have
+ * simply shifted from the pipe to the transcoder, while
+ * keeping their original offset. Thus we need PIPE_EDP_OFFSET
+ * to access such registers in transcoder EDP.
+ */
+#define PIPE_EDP_OFFSET 0x7f000
+
+#define _PIPE2(pipe, reg) (dev_priv->info.pipe_offsets[pipe] - \
+        dev_priv->info.pipe_offsets[PIPE_A] + (reg) + \
+        dev_priv->info.display_mmio_offset)
+
+#define PIPECONF(pipe) _PIPE2(pipe, _PIPEACONF)
+#define PIPEDSL(pipe)  _PIPE2(pipe, _PIPEADSL)
+#define PIPEFRAME(pipe) _PIPE2(pipe, _PIPEAFRAMEHIGH)
+#define PIPEFRAMEPIXEL(pipe)  _PIPE2(pipe, _PIPEAFRAMEPIXEL)
+#define PIPESTAT(pipe) _PIPE2(pipe, _PIPEASTAT)
 
 #define _PIPE_MISC_A                    0x70030
 #define _PIPE_MISC_B                    0x71030
@@ -3279,20 +3345,20 @@
 #define   PIPEMISC_DITHER_ENABLE        (1<<4)
 #define   PIPEMISC_DITHER_TYPE_MASK     (3<<2)
 #define   PIPEMISC_DITHER_TYPE_SP       (0<<2)
-#define PIPEMISC(pipe) _PIPE(pipe, _PIPE_MISC_A, _PIPE_MISC_B)
+#define PIPEMISC(pipe) _PIPE2(pipe, _PIPE_MISC_A)
 
 #define VLV_DPFLIPSTAT                          (VLV_DISPLAY_BASE + 0x70028)
 #define   PIPEB_LINE_COMPARE_INT_EN             (1<<29)
 #define   PIPEB_HLINE_INT_EN                    (1<<28)
 #define   PIPEB_VBLANK_INT_EN                   (1<<27)
-#define   SPRITED_FLIPDONE_INT_EN               (1<<26)
-#define   SPRITEC_FLIPDONE_INT_EN               (1<<25)
-#define   PLANEB_FLIPDONE_INT_EN                (1<<24)
+#define   SPRITED_FLIP_DONE_INT_EN              (1<<26)
+#define   SPRITEC_FLIP_DONE_INT_EN              (1<<25)
+#define   PLANEB_FLIP_DONE_INT_EN               (1<<24)
 #define   PIPEA_LINE_COMPARE_INT_EN             (1<<21)
 #define   PIPEA_HLINE_INT_EN                    (1<<20)
 #define   PIPEA_VBLANK_INT_EN                   (1<<19)
-#define   SPRITEB_FLIPDONE_INT_EN               (1<<18)
-#define   SPRITEA_FLIPDONE_INT_EN               (1<<17)
+#define   SPRITEB_FLIP_DONE_INT_EN              (1<<18)
+#define   SPRITEA_FLIP_DONE_INT_EN              (1<<17)
 #define   PLANEA_FLIPDONE_INT_EN                (1<<16)
 
 #define DPINVGTT                                (VLV_DISPLAY_BASE + 0x7002c) /* VLV only */
@@ -3323,7 +3389,7 @@
 #define   DSPARB_BEND_SHIFT     9 /* on 855 */
 #define   DSPARB_AEND_SHIFT     0
 
-#define DSPFW1                  (dev_priv->info->display_mmio_offset + 0x70034)
+#define DSPFW1                  (dev_priv->info.display_mmio_offset + 0x70034)
 #define   DSPFW_SR_SHIFT        23
 #define   DSPFW_SR_MASK         (0x1ff<<23)
 #define   DSPFW_CURSORB_SHIFT   16
@@ -3331,11 +3397,11 @@
 #define   DSPFW_PLANEB_SHIFT    8
 #define   DSPFW_PLANEB_MASK     (0x7f<<8)
 #define   DSPFW_PLANEA_MASK     (0x7f)
-#define DSPFW2                  (dev_priv->info->display_mmio_offset + 0x70038)
+#define DSPFW2                  (dev_priv->info.display_mmio_offset + 0x70038)
 #define   DSPFW_CURSORA_MASK    0x00003f00
 #define   DSPFW_CURSORA_SHIFT   8
 #define   DSPFW_PLANEC_MASK     (0x7f)
-#define DSPFW3                  (dev_priv->info->display_mmio_offset + 0x7003c)
+#define DSPFW3                  (dev_priv->info.display_mmio_offset + 0x7003c)
 #define   DSPFW_HPLL_SR_EN      (1<<31)
 #define   DSPFW_CURSOR_SR_SHIFT 24
 #define   PINEVIEW_SELF_REFRESH_EN      (1<<30)
@@ -3343,8 +3409,8 @@
 #define   DSPFW_HPLL_CURSOR_SHIFT       16
 #define   DSPFW_HPLL_CURSOR_MASK        (0x3f<<16)
 #define   DSPFW_HPLL_SR_MASK            (0x1ff)
-#define DSPFW4                  (dev_priv->info->display_mmio_offset + 0x70070)
-#define DSPFW7                  (dev_priv->info->display_mmio_offset + 0x7007c)
+#define DSPFW4                  (dev_priv->info.display_mmio_offset + 0x70070)
+#define DSPFW7                  (dev_priv->info.display_mmio_offset + 0x7007c)
 
 /* drain latency register values*/
 #define DRAIN_LATENCY_PRECISION_32      32
@@ -3468,12 +3534,12 @@
 #define   PIPE_PIXEL_MASK         0x00ffffff
 #define   PIPE_PIXEL_SHIFT        0
 /* GM45+ just has to be different */
-#define _PIPEA_FRMCOUNT_GM45    (dev_priv->info->display_mmio_offset + 0x70040)
-#define _PIPEA_FLIPCOUNT_GM45   (dev_priv->info->display_mmio_offset + 0x70044)
+#define _PIPEA_FRMCOUNT_GM45    (dev_priv->info.display_mmio_offset + 0x70040)
+#define _PIPEA_FLIPCOUNT_GM45   (dev_priv->info.display_mmio_offset + 0x70044)
 #define PIPE_FRMCOUNT_GM45(pipe) _PIPE(pipe, _PIPEA_FRMCOUNT_GM45, _PIPEB_FRMCOUNT_GM45)
 
 /* Cursor A & B regs */
-#define _CURACNTR               (dev_priv->info->display_mmio_offset + 0x70080)
+#define _CURACNTR               (dev_priv->info.display_mmio_offset + 0x70080)
 /* Old style CUR*CNTR flags (desktop 8xx) */
 #define   CURSOR_ENABLE         0x80000000
 #define   CURSOR_GAMMA_ENABLE   0x40000000
@@ -3489,23 +3555,27 @@
 /* New style CUR*CNTR flags */
 #define   CURSOR_MODE           0x27
 #define   CURSOR_MODE_DISABLE   0x00
+#define   CURSOR_MODE_128_32B_AX 0x02
+#define   CURSOR_MODE_256_32B_AX 0x03
 #define   CURSOR_MODE_64_32B_AX 0x07
+#define   CURSOR_MODE_128_ARGB_AX ((1 << 5) | CURSOR_MODE_128_32B_AX)
+#define   CURSOR_MODE_256_ARGB_AX ((1 << 5) | CURSOR_MODE_256_32B_AX)
 #define   CURSOR_MODE_64_ARGB_AX ((1 << 5) | CURSOR_MODE_64_32B_AX)
 #define   MCURSOR_PIPE_SELECT   (1 << 28)
 #define   MCURSOR_PIPE_A        0x00
 #define   MCURSOR_PIPE_B        (1 << 28)
 #define   MCURSOR_GAMMA_ENABLE  (1 << 26)
 #define   CURSOR_TRICKLE_FEED_DISABLE   (1 << 14)
-#define _CURABASE               (dev_priv->info->display_mmio_offset + 0x70084)
-#define _CURAPOS                (dev_priv->info->display_mmio_offset + 0x70088)
+#define _CURABASE               (dev_priv->info.display_mmio_offset + 0x70084)
+#define _CURAPOS                (dev_priv->info.display_mmio_offset + 0x70088)
 #define   CURSOR_POS_MASK       0x007FF
 #define   CURSOR_POS_SIGN       0x8000
 #define   CURSOR_X_SHIFT        0
 #define   CURSOR_Y_SHIFT        16
 #define CURSIZE                 0x700a0
-#define _CURBCNTR               (dev_priv->info->display_mmio_offset + 0x700c0)
-#define _CURBBASE               (dev_priv->info->display_mmio_offset + 0x700c4)
-#define _CURBPOS                (dev_priv->info->display_mmio_offset + 0x700c8)
+#define _CURBCNTR               (dev_priv->info.display_mmio_offset + 0x700c0)
+#define _CURBBASE               (dev_priv->info.display_mmio_offset + 0x700c4)
+#define _CURBPOS                (dev_priv->info.display_mmio_offset + 0x700c8)
 
 #define _CURBCNTR_IVB           0x71080
 #define _CURBBASE_IVB           0x71084
@@ -3520,7 +3590,7 @@
 #define CURPOS_IVB(pipe) _PIPE(pipe, _CURAPOS, _CURBPOS_IVB)
 
 /* Display A control */
-#define _DSPACNTR                (dev_priv->info->display_mmio_offset + 0x70180)
+#define _DSPACNTR                               0x70180
 #define   DISPLAY_PLANE_ENABLE                  (1<<31)
 #define   DISPLAY_PLANE_DISABLE                 0
 #define   DISPPLANE_GAMMA_ENABLE                (1<<30)
@@ -3554,25 +3624,25 @@
 #define   DISPPLANE_STEREO_POLARITY_SECOND      (1<<18)
 #define   DISPPLANE_TRICKLE_FEED_DISABLE        (1<<14) /* Ironlake */
 #define   DISPPLANE_TILED                       (1<<10)
-#define _DSPAADDR               (dev_priv->info->display_mmio_offset + 0x70184)
-#define _DSPASTRIDE             (dev_priv->info->display_mmio_offset + 0x70188)
-#define _DSPAPOS                (dev_priv->info->display_mmio_offset + 0x7018C) /* reserved */
-#define _DSPASIZE               (dev_priv->info->display_mmio_offset + 0x70190)
-#define _DSPASURF               (dev_priv->info->display_mmio_offset + 0x7019C) /* 965+ only */
-#define _DSPATILEOFF            (dev_priv->info->display_mmio_offset + 0x701A4) /* 965+ only */
-#define _DSPAOFFSET             (dev_priv->info->display_mmio_offset + 0x701A4) /* HSW */
-#define _DSPASURFLIVE           (dev_priv->info->display_mmio_offset + 0x701AC)
-
-#define DSPCNTR(plane) _PIPE(plane, _DSPACNTR, _DSPBCNTR)
-#define DSPADDR(plane) _PIPE(plane, _DSPAADDR, _DSPBADDR)
-#define DSPSTRIDE(plane) _PIPE(plane, _DSPASTRIDE, _DSPBSTRIDE)
-#define DSPPOS(plane) _PIPE(plane, _DSPAPOS, _DSPBPOS)
-#define DSPSIZE(plane) _PIPE(plane, _DSPASIZE, _DSPBSIZE)
-#define DSPSURF(plane) _PIPE(plane, _DSPASURF, _DSPBSURF)
-#define DSPTILEOFF(plane) _PIPE(plane, _DSPATILEOFF, _DSPBTILEOFF)
+#define _DSPAADDR                               0x70184
+#define _DSPASTRIDE                             0x70188
+#define _DSPAPOS                                0x7018C /* reserved */
+#define _DSPASIZE                               0x70190
+#define _DSPASURF                               0x7019C /* 965+ only */
+#define _DSPATILEOFF                            0x701A4 /* 965+ only */
+#define _DSPAOFFSET                             0x701A4 /* HSW */
+#define _DSPASURFLIVE                           0x701AC
+
+#define DSPCNTR(plane) _PIPE2(plane, _DSPACNTR)
+#define DSPADDR(plane) _PIPE2(plane, _DSPAADDR)
+#define DSPSTRIDE(plane) _PIPE2(plane, _DSPASTRIDE)
+#define DSPPOS(plane) _PIPE2(plane, _DSPAPOS)
+#define DSPSIZE(plane) _PIPE2(plane, _DSPASIZE)
+#define DSPSURF(plane) _PIPE2(plane, _DSPASURF)
+#define DSPTILEOFF(plane) _PIPE2(plane, _DSPATILEOFF)
 #define DSPLINOFF(plane) DSPADDR(plane)
-#define DSPOFFSET(plane) _PIPE(plane, _DSPAOFFSET, _DSPBOFFSET)
-#define DSPSURFLIVE(plane) _PIPE(plane, _DSPASURFLIVE, _DSPBSURFLIVE)
+#define DSPOFFSET(plane) _PIPE2(plane, _DSPAOFFSET)
+#define DSPSURFLIVE(plane) _PIPE2(plane, _DSPASURFLIVE)
 
 /* Display/Sprite base address macros */
 #define DISP_BASEADDR_MASK      (0xfffff000)
@@ -3580,44 +3650,44 @@
 #define I915_HI_DISPBASE(val)   (val & DISP_BASEADDR_MASK)
 
 /* VBIOS flags */
-#define SWF00                   (dev_priv->info->display_mmio_offset + 0x71410)
-#define SWF01                   (dev_priv->info->display_mmio_offset + 0x71414)
-#define SWF02                   (dev_priv->info->display_mmio_offset + 0x71418)
-#define SWF03                   (dev_priv->info->display_mmio_offset + 0x7141c)
-#define SWF04                   (dev_priv->info->display_mmio_offset + 0x71420)
-#define SWF05                   (dev_priv->info->display_mmio_offset + 0x71424)
-#define SWF06                   (dev_priv->info->display_mmio_offset + 0x71428)
-#define SWF10                   (dev_priv->info->display_mmio_offset + 0x70410)
-#define SWF11                   (dev_priv->info->display_mmio_offset + 0x70414)
-#define SWF14                   (dev_priv->info->display_mmio_offset + 0x71420)
-#define SWF30                   (dev_priv->info->display_mmio_offset + 0x72414)
-#define SWF31                   (dev_priv->info->display_mmio_offset + 0x72418)
-#define SWF32                   (dev_priv->info->display_mmio_offset + 0x7241c)
+#define SWF00                   (dev_priv->info.display_mmio_offset + 0x71410)
+#define SWF01                   (dev_priv->info.display_mmio_offset + 0x71414)
+#define SWF02                   (dev_priv->info.display_mmio_offset + 0x71418)
+#define SWF03                   (dev_priv->info.display_mmio_offset + 0x7141c)
+#define SWF04                   (dev_priv->info.display_mmio_offset + 0x71420)
+#define SWF05                   (dev_priv->info.display_mmio_offset + 0x71424)
+#define SWF06                   (dev_priv->info.display_mmio_offset + 0x71428)
+#define SWF10                   (dev_priv->info.display_mmio_offset + 0x70410)
+#define SWF11                   (dev_priv->info.display_mmio_offset + 0x70414)
+#define SWF14                   (dev_priv->info.display_mmio_offset + 0x71420)
+#define SWF30                   (dev_priv->info.display_mmio_offset + 0x72414)
+#define SWF31                   (dev_priv->info.display_mmio_offset + 0x72418)
+#define SWF32                   (dev_priv->info.display_mmio_offset + 0x7241c)
 
 /* Pipe B */
-#define _PIPEBDSL               (dev_priv->info->display_mmio_offset + 0x71000)
-#define _PIPEBCONF              (dev_priv->info->display_mmio_offset + 0x71008)
-#define _PIPEBSTAT              (dev_priv->info->display_mmio_offset + 0x71024)
+#define _PIPEBDSL               (dev_priv->info.display_mmio_offset + 0x71000)
+#define _PIPEBCONF              (dev_priv->info.display_mmio_offset + 0x71008)
+#define _PIPEBSTAT              (dev_priv->info.display_mmio_offset + 0x71024)
 #define _PIPEBFRAMEHIGH         0x71040
 #define _PIPEBFRAMEPIXEL        0x71044
-#define _PIPEB_FRMCOUNT_GM45    (dev_priv->info->display_mmio_offset + 0x71040)
-#define _PIPEB_FLIPCOUNT_GM45   (dev_priv->info->display_mmio_offset + 0x71044)
+#define _PIPEB_FRMCOUNT_GM45    (dev_priv->info.display_mmio_offset + 0x71040)
+#define _PIPEB_FLIPCOUNT_GM45   (dev_priv->info.display_mmio_offset + 0x71044)
 
 
 /* Display B control */
-#define _DSPBCNTR               (dev_priv->info->display_mmio_offset + 0x71180)
+#define _DSPBCNTR               (dev_priv->info.display_mmio_offset + 0x71180)
 #define   DISPPLANE_ALPHA_TRANS_ENABLE          (1<<15)
 #define   DISPPLANE_ALPHA_TRANS_DISABLE         0
 #define   DISPPLANE_SPRITE_ABOVE_DISPLAY        0
 #define   DISPPLANE_SPRITE_ABOVE_OVERLAY        (1)
-#define _DSPBADDR               (dev_priv->info->display_mmio_offset + 0x71184)
-#define _DSPBSTRIDE             (dev_priv->info->display_mmio_offset + 0x71188)
-#define _DSPBPOS                (dev_priv->info->display_mmio_offset + 0x7118C)
-#define _DSPBSIZE               (dev_priv->info->display_mmio_offset + 0x71190)
-#define _DSPBSURF               (dev_priv->info->display_mmio_offset + 0x7119C)
-#define _DSPBTILEOFF            (dev_priv->info->display_mmio_offset + 0x711A4)
-#define _DSPBOFFSET             (dev_priv->info->display_mmio_offset + 0x711A4)
-#define _DSPBSURFLIVE           (dev_priv->info->display_mmio_offset + 0x711AC)
+#define _DSPBADDR               (dev_priv->info.display_mmio_offset + 0x71184)
+#define _DSPBSTRIDE             (dev_priv->info.display_mmio_offset + 0x71188)
+#define _DSPBPOS                (dev_priv->info.display_mmio_offset + 0x7118C)
+#define _DSPBSIZE               (dev_priv->info.display_mmio_offset + 0x71190)
+#define _DSPBSURF               (dev_priv->info.display_mmio_offset + 0x7119C)
+#define _DSPBTILEOFF            (dev_priv->info.display_mmio_offset + 0x711A4)
+#define _DSPBOFFSET             (dev_priv->info.display_mmio_offset + 0x711A4)
+#define _DSPBSURFLIVE           (dev_priv->info.display_mmio_offset + 0x711AC)
 
 /* Sprite A control */
 #define _DVSACNTR               0x72180
@@ -3866,48 +3936,45 @@
 #define  FDI_PLL_FREQ_DISABLE_COUNT_LIMIT_MASK  0xff
 
 
-#define _PIPEA_DATA_M1           (dev_priv->info->display_mmio_offset + 0x60030)
+#define _PIPEA_DATA_M1          0x60030
 #define  PIPE_DATA_M1_OFFSET    0
-#define _PIPEA_DATA_N1           (dev_priv->info->display_mmio_offset + 0x60034)
+#define _PIPEA_DATA_N1          0x60034
 #define  PIPE_DATA_N1_OFFSET    0
 
-#define _PIPEA_DATA_M2           (dev_priv->info->display_mmio_offset + 0x60038)
+#define _PIPEA_DATA_M2          0x60038
 #define  PIPE_DATA_M2_OFFSET    0
-#define _PIPEA_DATA_N2           (dev_priv->info->display_mmio_offset + 0x6003c)
+#define _PIPEA_DATA_N2          0x6003c
 #define  PIPE_DATA_N2_OFFSET    0
 
-#define _PIPEA_LINK_M1           (dev_priv->info->display_mmio_offset + 0x60040)
+#define _PIPEA_LINK_M1          0x60040
 #define  PIPE_LINK_M1_OFFSET    0
-#define _PIPEA_LINK_N1           (dev_priv->info->display_mmio_offset + 0x60044)
+#define _PIPEA_LINK_N1          0x60044
 #define  PIPE_LINK_N1_OFFSET    0
 
-#define _PIPEA_LINK_M2           (dev_priv->info->display_mmio_offset + 0x60048)
+#define _PIPEA_LINK_M2          0x60048
 #define  PIPE_LINK_M2_OFFSET    0
-#define _PIPEA_LINK_N2           (dev_priv->info->display_mmio_offset + 0x6004c)
+#define _PIPEA_LINK_N2          0x6004c
 #define  PIPE_LINK_N2_OFFSET    0
 
 /* PIPEB timing regs are same start from 0x61000 */
 
-#define _PIPEB_DATA_M1           (dev_priv->info->display_mmio_offset + 0x61030)
-#define _PIPEB_DATA_N1           (dev_priv->info->display_mmio_offset + 0x61034)
-
-#define _PIPEB_DATA_M2           (dev_priv->info->display_mmio_offset + 0x61038)
-#define _PIPEB_DATA_N2           (dev_priv->info->display_mmio_offset + 0x6103c)
-
-#define _PIPEB_LINK_M1           (dev_priv->info->display_mmio_offset + 0x61040)
-#define _PIPEB_LINK_N1           (dev_priv->info->display_mmio_offset + 0x61044)
-
-#define _PIPEB_LINK_M2           (dev_priv->info->display_mmio_offset + 0x61048)
-#define _PIPEB_LINK_N2           (dev_priv->info->display_mmio_offset + 0x6104c)
-
-#define PIPE_DATA_M1(tran) _TRANSCODER(tran, _PIPEA_DATA_M1, _PIPEB_DATA_M1)
-#define PIPE_DATA_N1(tran) _TRANSCODER(tran, _PIPEA_DATA_N1, _PIPEB_DATA_N1)
-#define PIPE_DATA_M2(tran) _TRANSCODER(tran, _PIPEA_DATA_M2, _PIPEB_DATA_M2)
-#define PIPE_DATA_N2(tran) _TRANSCODER(tran, _PIPEA_DATA_N2, _PIPEB_DATA_N2)
-#define PIPE_LINK_M1(tran) _TRANSCODER(tran, _PIPEA_LINK_M1, _PIPEB_LINK_M1)
-#define PIPE_LINK_N1(tran) _TRANSCODER(tran, _PIPEA_LINK_N1, _PIPEB_LINK_N1)
-#define PIPE_LINK_M2(tran) _TRANSCODER(tran, _PIPEA_LINK_M2, _PIPEB_LINK_M2)
-#define PIPE_LINK_N2(tran) _TRANSCODER(tran, _PIPEA_LINK_N2, _PIPEB_LINK_N2)
+#define _PIPEB_DATA_M1          0x61030
+#define _PIPEB_DATA_N1          0x61034
+#define _PIPEB_DATA_M2          0x61038
+#define _PIPEB_DATA_N2          0x6103c
+#define _PIPEB_LINK_M1          0x61040
+#define _PIPEB_LINK_N1          0x61044
+#define _PIPEB_LINK_M2          0x61048
+#define _PIPEB_LINK_N2          0x6104c
+
+#define PIPE_DATA_M1(tran) _TRANSCODER2(tran, _PIPEA_DATA_M1)
+#define PIPE_DATA_N1(tran) _TRANSCODER2(tran, _PIPEA_DATA_N1)
+#define PIPE_DATA_M2(tran) _TRANSCODER2(tran, _PIPEA_DATA_M2)
+#define PIPE_DATA_N2(tran) _TRANSCODER2(tran, _PIPEA_DATA_N2)
+#define PIPE_LINK_M1(tran) _TRANSCODER2(tran, _PIPEA_LINK_M1)
+#define PIPE_LINK_N1(tran) _TRANSCODER2(tran, _PIPEA_LINK_N1)
+#define PIPE_LINK_M2(tran) _TRANSCODER2(tran, _PIPEA_LINK_M2)
+#define PIPE_LINK_N2(tran) _TRANSCODER2(tran, _PIPEA_LINK_N2)
 
 /* CPU panel fitter */
 /* IVB+ has 3 fitters, 0 is 7x5 capable, the other two only 3x3 */
@@ -4084,13 +4151,14 @@
 #define  ILK_ELPIN_409_SELECT   (1 << 25)
 #define  ILK_DPARB_GATE (1<<22)
 #define  ILK_VSDPFD_FULL        (1<<21)
-#define ILK_DISPLAY_CHICKEN_FUSES       0x42014
-#define  ILK_INTERNAL_GRAPHICS_DISABLE  (1<<31)
-#define  ILK_INTERNAL_DISPLAY_DISABLE   (1<<30)
-#define  ILK_DISPLAY_DEBUG_DISABLE      (1<<29)
-#define  ILK_HDCP_DISABLE               (1<<25)
-#define  ILK_eDP_A_DISABLE              (1<<24)
-#define  ILK_DESKTOP                    (1<<23)
+#define FUSE_STRAP                      0x42014
+#define  ILK_INTERNAL_GRAPHICS_DISABLE  (1 << 31)
+#define  ILK_INTERNAL_DISPLAY_DISABLE   (1 << 30)
+#define  ILK_DISPLAY_DEBUG_DISABLE      (1 << 29)
+#define  ILK_HDCP_DISABLE               (1 << 25)
+#define  ILK_eDP_A_DISABLE              (1 << 24)
+#define  HSW_CDCLK_LIMIT                (1 << 24)
+#define  ILK_DESKTOP                    (1 << 23)
 
 #define ILK_DSPCLK_GATE_D                       0x42020
 #define   ILK_VRHUNIT_CLOCK_GATE_DISABLE        (1 << 28)
@@ -4109,7 +4177,8 @@
 
 #define _CHICKEN_PIPESL_1_A     0x420b0
 #define _CHICKEN_PIPESL_1_B     0x420b4
-#define  DPRS_MASK_VBLANK_SRD   (1 << 0)
+#define  HSW_FBCQ_DIS                   (1 << 22)
+#define  BDW_DPRS_MASK_VBLANK_SRD       (1 << 0)
 #define CHICKEN_PIPESL_1(pipe) _PIPE(pipe, _CHICKEN_PIPESL_1_A, _CHICKEN_PIPESL_1_B)
 
 #define DISP_ARB_CTL    0x45000
@@ -4120,6 +4189,8 @@
 #define GEN7_MSG_CTL    0x45010
 #define  WAIT_FOR_PCH_RESET_ACK         (1<<1)
 #define  WAIT_FOR_PCH_FLR_ACK           (1<<0)
+#define HSW_NDE_RSTWRN_OPT      0x46408
+#define  RESET_PCH_HANDSHAKE_ENABLE     (1<<4)
 
 /* GEN7 chicken */
 #define GEN7_COMMON_SLICE_CHICKEN1              0x7010
@@ -4127,8 +4198,11 @@
 #define COMMON_SLICE_CHICKEN2                   0x7014
 # define GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE   (1<<0)
 
+#define GEN7_L3SQCREG1                          0xB010
+#define  VLV_B0_WA_L3SQCREG1_VALUE              0x00D30000
+
 #define GEN7_L3CNTLREG1                         0xB01C
-#define  GEN7_WA_FOR_GEN7_L3_CONTROL                    0x3C4FFF8C
+#define  GEN7_WA_FOR_GEN7_L3_CONTROL                    0x3C47FF8C
 #define  GEN7_L3AGDIS                           (1<<19)
 
 #define GEN7_L3_CHICKEN_MODE_REGISTER           0xB030
@@ -4148,9 +4222,6 @@
 #define HSW_SCRATCH1                            0xb038
 #define  HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE   (1<<27)
 
-#define HSW_FUSE_STRAP          0x42014
-#define  HSW_CDCLK_LIMIT        (1 << 24)
-
 /* PCH */
 
 /* south display engine interrupt: IBX */
@@ -4436,24 +4507,24 @@
 #define HSW_VIDEO_DIP_GCP_B             0x61210
 
 #define HSW_TVIDEO_DIP_CTL(trans) \
-         _TRANSCODER(trans, HSW_VIDEO_DIP_CTL_A, HSW_VIDEO_DIP_CTL_B)
+         _TRANSCODER2(trans, HSW_VIDEO_DIP_CTL_A)
 #define HSW_TVIDEO_DIP_AVI_DATA(trans) \
-         _TRANSCODER(trans, HSW_VIDEO_DIP_AVI_DATA_A, HSW_VIDEO_DIP_AVI_DATA_B)
+         _TRANSCODER2(trans, HSW_VIDEO_DIP_AVI_DATA_A)
 #define HSW_TVIDEO_DIP_VS_DATA(trans) \
-         _TRANSCODER(trans, HSW_VIDEO_DIP_VS_DATA_A, HSW_VIDEO_DIP_VS_DATA_B)
+         _TRANSCODER2(trans, HSW_VIDEO_DIP_VS_DATA_A)
 #define HSW_TVIDEO_DIP_SPD_DATA(trans) \
-         _TRANSCODER(trans, HSW_VIDEO_DIP_SPD_DATA_A, HSW_VIDEO_DIP_SPD_DATA_B)
+         _TRANSCODER2(trans, HSW_VIDEO_DIP_SPD_DATA_A)
 #define HSW_TVIDEO_DIP_GCP(trans) \
-        _TRANSCODER(trans, HSW_VIDEO_DIP_GCP_A, HSW_VIDEO_DIP_GCP_B)
+        _TRANSCODER2(trans, HSW_VIDEO_DIP_GCP_A)
 #define HSW_TVIDEO_DIP_VSC_DATA(trans) \
-         _TRANSCODER(trans, HSW_VIDEO_DIP_VSC_DATA_A, HSW_VIDEO_DIP_VSC_DATA_B)
+         _TRANSCODER2(trans, HSW_VIDEO_DIP_VSC_DATA_A)
 
 #define HSW_STEREO_3D_CTL_A     0x70020
 #define   S3D_ENABLE            (1<<31)
 #define HSW_STEREO_3D_CTL_B     0x71020
 
 #define HSW_STEREO_3D_CTL(trans) \
-        _TRANSCODER(trans, HSW_STEREO_3D_CTL_A, HSW_STEREO_3D_CTL_A)
+        _PIPE2(trans, HSW_STEREO_3D_CTL_A)
 
 #define _PCH_TRANS_HTOTAL_B          0xe1000
 #define _PCH_TRANS_HBLANK_B          0xe1004
@@ -4865,6 +4936,9 @@
 #define GEN7_UCGCTL4                            0x940c
 #define  GEN7_L3BANK2X_CLOCK_GATE_DISABLE       (1<<25)
 
+#define GEN8_UCGCTL6                            0x9430
+#define   GEN8_SDEUNIT_CLOCK_GATE_DISABLE       (1<<14)
+
 #define GEN6_RPNSWREQ                           0xA008
 #define   GEN6_TURBO_DISABLE                    (1<<31)
 #define   GEN6_FREQUENCY(x)                     ((x)<<25)
@@ -4945,6 +5019,10 @@
                                                  GEN6_PM_RP_DOWN_THRESHOLD | \
                                                  GEN6_PM_RP_DOWN_TIMEOUT)
 
+#define VLV_GTLC_SURVIVABILITY_REG              0x130098
+#define VLV_GFX_CLK_STATUS_BIT                  (1<<3)
+#define VLV_GFX_CLK_FORCE_ON_BIT                (1<<2)
+
 #define GEN6_GT_GFX_RC6_LOCKED                  0x138104
 #define VLV_COUNTER_CONTROL                     0x138104
 #define   VLV_COUNT_RANGE_HIGH                  (1<<15)
@@ -5006,6 +5084,10 @@
 #define   GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE   (1<<10)
 #define   GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE  (1<<3)
 
+#define GEN8_ROW_CHICKEN                0xe4f0
+#define   PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE (1<<8)
+#define   STALL_DOP_GATING_DISABLE              (1<<5)
+
 #define GEN7_ROW_CHICKEN2               0xe4f4
 #define GEN7_ROW_CHICKEN2_GT2           0xf4f4
 #define   DOP_CLOCK_GATING_DISABLE      (1<<0)
@@ -5017,7 +5099,7 @@
 #define   GEN8_CENTROID_PIXEL_OPT_DIS   (1<<8)
 #define   GEN8_SAMPLER_POWER_BYPASS_DIS (1<<1)
 
-#define G4X_AUD_VID_DID                 (dev_priv->info->display_mmio_offset + 0x62020)
+#define G4X_AUD_VID_DID                 (dev_priv->info.display_mmio_offset + 0x62020)
 #define INTEL_AUDIO_DEVCL               0x808629FB
 #define INTEL_AUDIO_DEVBLC              0x80862801
 #define INTEL_AUDIO_DEVCTG              0x80862802
@@ -5178,8 +5260,8 @@
 #define TRANS_DDI_FUNC_CTL_B            0x61400
 #define TRANS_DDI_FUNC_CTL_C            0x62400
 #define TRANS_DDI_FUNC_CTL_EDP          0x6F400
-#define TRANS_DDI_FUNC_CTL(tran) _TRANSCODER(tran, TRANS_DDI_FUNC_CTL_A, \
-                                                   TRANS_DDI_FUNC_CTL_B)
+#define TRANS_DDI_FUNC_CTL(tran) _TRANSCODER2(tran, TRANS_DDI_FUNC_CTL_A)
+
 #define  TRANS_DDI_FUNC_ENABLE          (1<<31)
 /* Those bits are ignored by pipe EDP since it can only connect to DDI A */
 #define  TRANS_DDI_PORT_MASK            (7<<28)
@@ -5311,8 +5393,12 @@
 #define  SPLL_PLL_ENABLE                (1<<31)
 #define  SPLL_PLL_SSC                   (1<<28)
 #define  SPLL_PLL_NON_SSC               (2<<28)
+#define  SPLL_PLL_LCPLL                 (3<<28)
+#define  SPLL_PLL_REF_MASK              (3<<28)
 #define  SPLL_PLL_FREQ_810MHz           (0<<26)
 #define  SPLL_PLL_FREQ_1350MHz          (1<<26)
+#define  SPLL_PLL_FREQ_2700MHz          (2<<26)
+#define  SPLL_PLL_FREQ_MASK             (3<<26)
 
 /* WRPLL */
 #define WRPLL_CTL1                      0x46040
@@ -5323,8 +5409,13 @@
 #define  WRPLL_PLL_SELECT_LCPLL_2700    (0x03<<28)
 /* WRPLL divider programming */
 #define  WRPLL_DIVIDER_REFERENCE(x)     ((x)<<0)
+#define  WRPLL_DIVIDER_REF_MASK         (0xff)
 #define  WRPLL_DIVIDER_POST(x)          ((x)<<8)
+#define  WRPLL_DIVIDER_POST_MASK        (0x3f<<8)
+#define  WRPLL_DIVIDER_POST_SHIFT       8
 #define  WRPLL_DIVIDER_FEEDBACK(x)      ((x)<<16)
+#define  WRPLL_DIVIDER_FB_SHIFT         16
+#define  WRPLL_DIVIDER_FB_MASK          (0xff<<16)
 
 /* Port clock selection */
 #define PORT_CLK_SEL_A                  0x46100
@@ -5337,6 +5428,7 @@
 #define  PORT_CLK_SEL_WRPLL1            (4<<29)
 #define  PORT_CLK_SEL_WRPLL2            (5<<29)
 #define  PORT_CLK_SEL_NONE              (7<<29)
+#define  PORT_CLK_SEL_MASK              (7<<29)
 
 /* Transcoder clock selection */
 #define TRANS_CLK_SEL_A                 0x46140
@@ -5346,10 +5438,12 @@
 #define  TRANS_CLK_SEL_DISABLED         (0x0<<29)
 #define  TRANS_CLK_SEL_PORT(x)          ((x+1)<<29)
 
-#define _TRANSA_MSA_MISC                0x60410
-#define _TRANSB_MSA_MISC                0x61410
-#define TRANS_MSA_MISC(tran) _TRANSCODER(tran, _TRANSA_MSA_MISC, \
-                                               _TRANSB_MSA_MISC)
+#define TRANSA_MSA_MISC                 0x60410
+#define TRANSB_MSA_MISC                 0x61410
+#define TRANSC_MSA_MISC                 0x62410
+#define TRANS_EDP_MSA_MISC              0x6f410
+#define TRANS_MSA_MISC(tran) _TRANSCODER2(tran, TRANSA_MSA_MISC)
+
 #define  TRANS_MSA_SYNC_CLK             (1<<0)
 #define  TRANS_MSA_6_BPC                (0<<5)
 #define  TRANS_MSA_8_BPC                (1<<5)
@@ -5389,6 +5483,8 @@
 
 /* SFUSE_STRAP */
 #define SFUSE_STRAP                     0xc2014
+#define  SFUSE_STRAP_FUSE_LOCK          (1<<13)
+#define  SFUSE_STRAP_DISPLAY_DISABLED   (1<<7)
 #define  SFUSE_STRAP_DDIB_DETECTED      (1<<2)
 #define  SFUSE_STRAP_DDIC_DETECTED      (1<<1)
 #define  SFUSE_STRAP_DDID_DETECTED      (1<<0)
@@ -5857,4 +5953,12 @@
 #define MIPI_READ_DATA_VALID(pipe)      _PIPE(pipe, _MIPIA_READ_DATA_VALID, _MIPIB_READ_DATA_VALID)
 #define  READ_DATA_VALID(n)                             (1 << (n))
 
+/* For UMS only (deprecated): */
+#define _PALETTE_A (dev_priv->info.display_mmio_offset + 0xa000)
+#define _PALETTE_B (dev_priv->info.display_mmio_offset + 0xa800)
+#define _DPLL_A (dev_priv->info.display_mmio_offset + 0x6014)
+#define _DPLL_B (dev_priv->info.display_mmio_offset + 0x6018)
+#define _DPLL_A_MD (dev_priv->info.display_mmio_offset + 0x601c)
+#define _DPLL_B_MD (dev_priv->info.display_mmio_offset + 0x6020)
+
 #endif /* _I915_REG_H_ */
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 8150fdc08d49..56785e8fb2eb 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -236,19 +236,9 @@ static void i915_save_display(struct drm_device *dev)
                 dev_priv->regfile.savePP_DIVISOR = I915_READ(PP_DIVISOR);
         }
 
-        /* Only regfile.save FBC state on the platform that supports FBC */
-        if (HAS_FBC(dev)) {
-                if (HAS_PCH_SPLIT(dev)) {
-                        dev_priv->regfile.saveDPFC_CB_BASE = I915_READ(ILK_DPFC_CB_BASE);
-                } else if (IS_GM45(dev)) {
-                        dev_priv->regfile.saveDPFC_CB_BASE = I915_READ(DPFC_CB_BASE);
-                } else {
-                        dev_priv->regfile.saveFBC_CFB_BASE = I915_READ(FBC_CFB_BASE);
-                        dev_priv->regfile.saveFBC_LL_BASE = I915_READ(FBC_LL_BASE);
-                        dev_priv->regfile.saveFBC_CONTROL2 = I915_READ(FBC_CONTROL2);
-                        dev_priv->regfile.saveFBC_CONTROL = I915_READ(FBC_CONTROL);
-                }
-        }
+        /* save FBC interval */
+        if (HAS_FBC(dev) && INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev))
+                dev_priv->regfile.saveFBC_CONTROL = I915_READ(FBC_CONTROL);
 
         if (!drm_core_check_feature(dev, DRIVER_MODESET))
                 i915_save_vga(dev);
@@ -300,18 +290,10 @@ static void i915_restore_display(struct drm_device *dev)
 
         /* only restore FBC info on the platform that supports FBC*/
         intel_disable_fbc(dev);
-        if (HAS_FBC(dev)) {
-                if (HAS_PCH_SPLIT(dev)) {
-                        I915_WRITE(ILK_DPFC_CB_BASE, dev_priv->regfile.saveDPFC_CB_BASE);
-                } else if (IS_GM45(dev)) {
-                        I915_WRITE(DPFC_CB_BASE, dev_priv->regfile.saveDPFC_CB_BASE);
-                } else {
-                        I915_WRITE(FBC_CFB_BASE, dev_priv->regfile.saveFBC_CFB_BASE);
-                        I915_WRITE(FBC_LL_BASE, dev_priv->regfile.saveFBC_LL_BASE);
-                        I915_WRITE(FBC_CONTROL2, dev_priv->regfile.saveFBC_CONTROL2);
-                        I915_WRITE(FBC_CONTROL, dev_priv->regfile.saveFBC_CONTROL);
-                }
-        }
+
+        /* restore FBC interval */
+        if (HAS_FBC(dev) && INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev))
+                I915_WRITE(FBC_CONTROL, dev_priv->regfile.saveFBC_CONTROL);
 
         if (!drm_core_check_feature(dev, DRIVER_MODESET))
                 i915_restore_vga(dev);
@@ -324,10 +306,6 @@ int i915_save_state(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int i;
 
-        if (INTEL_INFO(dev)->gen <= 4)
-                pci_read_config_byte(dev->pdev, LBB,
-                                     &dev_priv->regfile.saveLBB);
-
         mutex_lock(&dev->struct_mutex);
 
         i915_save_display(dev);
@@ -377,10 +355,6 @@ int i915_restore_state(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int i;
 
-        if (INTEL_INFO(dev)->gen <= 4)
-                pci_write_config_byte(dev->pdev, LBB,
-                                      dev_priv->regfile.saveLBB);
-
         mutex_lock(&dev->struct_mutex);
 
         i915_gem_restore_fences(dev);
diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
index 33bcae314bf8..9c57029f6f4b 100644
--- a/drivers/gpu/drm/i915/i915_sysfs.c
+++ b/drivers/gpu/drm/i915/i915_sysfs.c
@@ -269,7 +269,7 @@ static ssize_t gt_cur_freq_mhz_show(struct device *kdev,
                 freq = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
                 ret = vlv_gpu_freq(dev_priv, (freq >> 8) & 0xff);
         } else {
-                ret = dev_priv->rps.cur_delay * GT_FREQUENCY_MULTIPLIER;
+                ret = dev_priv->rps.cur_freq * GT_FREQUENCY_MULTIPLIER;
         }
         mutex_unlock(&dev_priv->rps.hw_lock);
 
@@ -284,7 +284,7 @@ static ssize_t vlv_rpe_freq_mhz_show(struct device *kdev,
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         return snprintf(buf, PAGE_SIZE, "%d\n",
-                        vlv_gpu_freq(dev_priv, dev_priv->rps.rpe_delay));
+                        vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq));
 }
 
 static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute *attr, char *buf)
@@ -298,9 +298,9 @@ static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute
 
         mutex_lock(&dev_priv->rps.hw_lock);
         if (IS_VALLEYVIEW(dev_priv->dev))
-                ret = vlv_gpu_freq(dev_priv, dev_priv->rps.max_delay);
+                ret = vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq_softlimit);
         else
-                ret = dev_priv->rps.max_delay * GT_FREQUENCY_MULTIPLIER;
+                ret = dev_priv->rps.max_freq_softlimit * GT_FREQUENCY_MULTIPLIER;
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         return snprintf(buf, PAGE_SIZE, "%d\n", ret);
@@ -313,7 +313,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
         struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 val, rp_state_cap, hw_max, hw_min, non_oc_max;
+        u32 val;
         ssize_t ret;
 
         ret = kstrtou32(buf, 0, &val);
@@ -324,38 +324,34 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
 
         mutex_lock(&dev_priv->rps.hw_lock);
 
-        if (IS_VALLEYVIEW(dev_priv->dev)) {
+        if (IS_VALLEYVIEW(dev_priv->dev))
                 val = vlv_freq_opcode(dev_priv, val);
-
-                hw_max = valleyview_rps_max_freq(dev_priv);
-                hw_min = valleyview_rps_min_freq(dev_priv);
-                non_oc_max = hw_max;
-        } else {
+        else
                 val /= GT_FREQUENCY_MULTIPLIER;
 
-                rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
-                hw_max = dev_priv->rps.hw_max;
-                non_oc_max = (rp_state_cap & 0xff);
-                hw_min = ((rp_state_cap & 0xff0000) >> 16);
-        }
-
-        if (val < hw_min || val > hw_max ||
-            val < dev_priv->rps.min_delay) {
+        if (val < dev_priv->rps.min_freq ||
+            val > dev_priv->rps.max_freq ||
+            val < dev_priv->rps.min_freq_softlimit) {
                 mutex_unlock(&dev_priv->rps.hw_lock);
                 return -EINVAL;
         }
 
-        if (val > non_oc_max)
+        if (val > dev_priv->rps.rp0_freq)
                 DRM_DEBUG("User requested overclocking to %d\n",
                           val * GT_FREQUENCY_MULTIPLIER);
 
-        dev_priv->rps.max_delay = val;
+        dev_priv->rps.max_freq_softlimit = val;
 
-        if (dev_priv->rps.cur_delay > val) {
+        if (dev_priv->rps.cur_freq > val) {
                 if (IS_VALLEYVIEW(dev))
                         valleyview_set_rps(dev, val);
                 else
                         gen6_set_rps(dev, val);
+        } else if (!IS_VALLEYVIEW(dev)) {
+                /* We still need gen6_set_rps to process the new max_delay and
+                 * update the interrupt limits even though frequency request is
+                 * unchanged. */
+                gen6_set_rps(dev, dev_priv->rps.cur_freq);
         }
 
         mutex_unlock(&dev_priv->rps.hw_lock);
@@ -374,9 +370,9 @@ static ssize_t gt_min_freq_mhz_show(struct device *kdev, struct device_attribute
 
         mutex_lock(&dev_priv->rps.hw_lock);
         if (IS_VALLEYVIEW(dev_priv->dev))
-                ret = vlv_gpu_freq(dev_priv, dev_priv->rps.min_delay);
+                ret = vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq_softlimit);
         else
-                ret = dev_priv->rps.min_delay * GT_FREQUENCY_MULTIPLIER;
+                ret = dev_priv->rps.min_freq_softlimit * GT_FREQUENCY_MULTIPLIER;
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         return snprintf(buf, PAGE_SIZE, "%d\n", ret);
@@ -389,7 +385,7 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev,
         struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 val, rp_state_cap, hw_max, hw_min;
+        u32 val;
         ssize_t ret;
 
         ret = kstrtou32(buf, 0, &val);
@@ -400,31 +396,30 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev,
 
         mutex_lock(&dev_priv->rps.hw_lock);
 
-        if (IS_VALLEYVIEW(dev)) {
+        if (IS_VALLEYVIEW(dev))
                 val = vlv_freq_opcode(dev_priv, val);
-
-                hw_max = valleyview_rps_max_freq(dev_priv);
-                hw_min = valleyview_rps_min_freq(dev_priv);
-        } else {
+        else
                 val /= GT_FREQUENCY_MULTIPLIER;
 
-                rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
-                hw_max = dev_priv->rps.hw_max;
-                hw_min = ((rp_state_cap & 0xff0000) >> 16);
-        }
-
-        if (val < hw_min || val > hw_max || val > dev_priv->rps.max_delay) {
+        if (val < dev_priv->rps.min_freq ||
+            val > dev_priv->rps.max_freq ||
+            val > dev_priv->rps.max_freq_softlimit) {
                 mutex_unlock(&dev_priv->rps.hw_lock);
                 return -EINVAL;
         }
 
-        dev_priv->rps.min_delay = val;
+        dev_priv->rps.min_freq_softlimit = val;
 
-        if (dev_priv->rps.cur_delay < val) {
+        if (dev_priv->rps.cur_freq < val) {
                 if (IS_VALLEYVIEW(dev))
                         valleyview_set_rps(dev, val);
                 else
                         gen6_set_rps(dev, val);
+        } else if (!IS_VALLEYVIEW(dev)) {
+                /* We still need gen6_set_rps to process the new min_delay and
+                 * update the interrupt limits even though frequency request is
+                 * unchanged. */
+                gen6_set_rps(dev, dev_priv->rps.cur_freq);
         }
 
         mutex_unlock(&dev_priv->rps.hw_lock);
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index 6e580c98dede..23c26f1f8b37 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -34,15 +34,15 @@ TRACE_EVENT(i915_gem_object_create,
 );
 
 TRACE_EVENT(i915_vma_bind,
-            TP_PROTO(struct i915_vma *vma, bool mappable),
-            TP_ARGS(vma, mappable),
+            TP_PROTO(struct i915_vma *vma, unsigned flags),
+            TP_ARGS(vma, flags),
 
             TP_STRUCT__entry(
                              __field(struct drm_i915_gem_object *, obj)
                              __field(struct i915_address_space *, vm)
                              __field(u32, offset)
                              __field(u32, size)
-                             __field(bool, mappable)
+                             __field(unsigned, flags)
                              ),
 
             TP_fast_assign(
@@ -50,12 +50,12 @@ TRACE_EVENT(i915_vma_bind,
                            __entry->vm = vma->vm;
                            __entry->offset = vma->node.start;
                            __entry->size = vma->node.size;
-                           __entry->mappable = mappable;
+                           __entry->flags = flags;
                            ),
 
             TP_printk("obj=%p, offset=%08x size=%x%s vm=%p",
                       __entry->obj, __entry->offset, __entry->size,
-                      __entry->mappable ? ", mappable" : "",
+                      __entry->flags & PIN_MAPPABLE ? ", mappable" : "",
                       __entry->vm)
 );
 
@@ -196,26 +196,26 @@ DEFINE_EVENT(i915_gem_object, i915_gem_object_destroy,
 );
 
 TRACE_EVENT(i915_gem_evict,
-            TP_PROTO(struct drm_device *dev, u32 size, u32 align, bool mappable),
-            TP_ARGS(dev, size, align, mappable),
+            TP_PROTO(struct drm_device *dev, u32 size, u32 align, unsigned flags),
+            TP_ARGS(dev, size, align, flags),
 
             TP_STRUCT__entry(
                              __field(u32, dev)
                              __field(u32, size)
                              __field(u32, align)
-                             __field(bool, mappable)
+                             __field(unsigned, flags)
                             ),
 
             TP_fast_assign(
                            __entry->dev = dev->primary->index;
                            __entry->size = size;
                            __entry->align = align;
-                           __entry->mappable = mappable;
+                           __entry->flags = flags;
                           ),
 
             TP_printk("dev=%d, size=%d, align=%d %s",
                       __entry->dev, __entry->size, __entry->align,
-                      __entry->mappable ? ", mappable" : "")
+                      __entry->flags & PIN_MAPPABLE ? ", mappable" : "")
 );
 
 TRACE_EVENT(i915_gem_evict_everything,
@@ -238,14 +238,16 @@ TRACE_EVENT(i915_gem_evict_vm,
             TP_ARGS(vm),
 
             TP_STRUCT__entry(
+                             __field(u32, dev)
                              __field(struct i915_address_space *, vm)
                             ),
 
             TP_fast_assign(
+                           __entry->dev = vm->dev->primary->index;
                            __entry->vm = vm;
                           ),
 
-            TP_printk("dev=%d, vm=%p", __entry->vm->dev->primary->index, __entry->vm)
+            TP_printk("dev=%d, vm=%p", __entry->dev, __entry->vm)
 );
 
 TRACE_EVENT(i915_gem_ring_sync_to,
diff --git a/drivers/gpu/drm/i915/i915_ums.c b/drivers/gpu/drm/i915/i915_ums.c
index caa18e855815..480da593e6c0 100644
--- a/drivers/gpu/drm/i915/i915_ums.c
+++ b/drivers/gpu/drm/i915/i915_ums.c
@@ -271,6 +271,10 @@ void i915_save_display_reg(struct drm_device *dev)
         /* FIXME: regfile.save TV & SDVO state */
 
         /* Backlight */
+        if (INTEL_INFO(dev)->gen <= 4)
+                pci_read_config_byte(dev->pdev, PCI_LBPC,
+                                     &dev_priv->regfile.saveLBB);
+
         if (HAS_PCH_SPLIT(dev)) {
                 dev_priv->regfile.saveBLC_PWM_CTL = I915_READ(BLC_PWM_PCH_CTL1);
                 dev_priv->regfile.saveBLC_PWM_CTL2 = I915_READ(BLC_PWM_PCH_CTL2);
@@ -293,6 +297,10 @@ void i915_restore_display_reg(struct drm_device *dev)
         int i;
 
         /* Backlight */
+        if (INTEL_INFO(dev)->gen <= 4)
+                pci_write_config_byte(dev->pdev, PCI_LBPC,
+                                      dev_priv->regfile.saveLBB);
+
         if (HAS_PCH_SPLIT(dev)) {
                 I915_WRITE(BLC_PWM_PCH_CTL1, dev_priv->regfile.saveBLC_PWM_CTL);
                 I915_WRITE(BLC_PWM_PCH_CTL2, dev_priv->regfile.saveBLC_PWM_CTL2);
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index f22041973f3a..4867f4cc0938 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -259,7 +259,7 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv,
                         downclock = dvo_timing->clock;
         }
 
-        if (downclock < panel_dvo_timing->clock && i915_lvds_downclock) {
+        if (downclock < panel_dvo_timing->clock && i915.lvds_downclock) {
                 dev_priv->lvds_downclock_avail = 1;
                 dev_priv->lvds_downclock = downclock * 10;
                 DRM_DEBUG_KMS("LVDS downclock is found in VBT. "
@@ -318,7 +318,7 @@ parse_sdvo_panel_data(struct drm_i915_private *dev_priv,
         struct drm_display_mode *panel_fixed_mode;
         int index;
 
-        index = i915_vbt_sdvo_panel_type;
+        index = i915.vbt_sdvo_panel_type;
         if (index == -2) {
                 DRM_DEBUG_KMS("Ignore SDVO panel mode from BIOS VBT tables.\n");
                 return;
@@ -599,14 +599,14 @@ parse_mipi(struct drm_i915_private *dev_priv, struct bdb_header *bdb)
 {
         struct bdb_mipi *mipi;
 
-        mipi = find_section(bdb, BDB_MIPI);
+        mipi = find_section(bdb, BDB_MIPI_CONFIG);
         if (!mipi) {
                 DRM_DEBUG_KMS("No MIPI BDB found");
                 return;
         }
 
         /* XXX: add more info */
-        dev_priv->vbt.dsi.panel_id = mipi->panel_id;
+        dev_priv->vbt.dsi.panel_id = MIPI_DSI_GENERIC_PANEL_ID;
 }
 
 static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
diff --git a/drivers/gpu/drm/i915/intel_bios.h b/drivers/gpu/drm/i915/intel_bios.h
index 282de5e9f39d..83b7629e4367 100644
--- a/drivers/gpu/drm/i915/intel_bios.h
+++ b/drivers/gpu/drm/i915/intel_bios.h
@@ -104,7 +104,8 @@ struct vbios_data {
 #define BDB_LVDS_LFP_DATA        42
 #define BDB_LVDS_BACKLIGHT       43
 #define BDB_LVDS_POWER           44
-#define BDB_MIPI                 50
+#define BDB_MIPI_CONFIG          52
+#define BDB_MIPI_SEQUENCE        53
 #define BDB_SKIP                254 /* VBIOS private block, ignore */
 
 struct bdb_general_features {
@@ -711,44 +712,159 @@ int intel_parse_bios(struct drm_device *dev);
 #define DVO_PORT_DPD    9
 #define DVO_PORT_DPA    10
 
-/* MIPI DSI panel info */
-struct bdb_mipi {
-        u16 panel_id;
-        u16 bridge_revision;
-
-        /* General params */
-        u32 dithering:1;
-        u32 bpp_pixel_format:1;
-        u32 rsvd1:1;
-        u32 dphy_valid:1;
-        u32 resvd2:28;
+/* Block 52 contains MIPI Panel info
+ * 6 such enteries will there. Index into correct
+ * entery is based on the panel_index in #40 LFP
+ */
+#define MAX_MIPI_CONFIGURATIONS 6
 
-        u16 port_info;
-        u16 rsvd3:2;
-        u16 num_lanes:2;
-        u16 rsvd4:12;
+#define MIPI_DSI_UNDEFINED_PANEL_ID     0
+#define MIPI_DSI_GENERIC_PANEL_ID       1
 
-        /* DSI config */
-        u16 virt_ch_num:2;
-        u16 vtm:2;
-        u16 rsvd5:12;
+struct mipi_config {
+        u16 panel_id;
 
-        u32 dsi_clock;
+        /* General Params */
+        u32 enable_dithering:1;
+        u32 rsvd1:1;
+        u32 is_bridge:1;
+
+        u32 panel_arch_type:2;
+        u32 is_cmd_mode:1;
+
+#define NON_BURST_SYNC_PULSE    0x1
+#define NON_BURST_SYNC_EVENTS   0x2
+#define BURST_MODE              0x3
+        u32 video_transfer_mode:2;
+
+        u32 cabc_supported:1;
+        u32 pwm_blc:1;
+
+        /* Bit 13:10 */
+#define PIXEL_FORMAT_RGB565                     0x1
+#define PIXEL_FORMAT_RGB666                     0x2
+#define PIXEL_FORMAT_RGB666_LOOSELY_PACKED      0x3
+#define PIXEL_FORMAT_RGB888                     0x4
+        u32 videomode_color_format:4;
+
+        /* Bit 15:14 */
+#define ENABLE_ROTATION_0       0x0
+#define ENABLE_ROTATION_90      0x1
+#define ENABLE_ROTATION_180     0x2
+#define ENABLE_ROTATION_270     0x3
+        u32 rotation:2;
+        u32 bta_enabled:1;
+        u32 rsvd2:15;
+
+        /* 2 byte Port Description */
+#define DUAL_LINK_NOT_SUPPORTED 0
+#define DUAL_LINK_FRONT_BACK    1
+#define DUAL_LINK_PIXEL_ALT     2
+        u16 dual_link:2;
+        u16 lane_cnt:2;
+        u16 rsvd3:12;
+
+        u16 rsvd4;
+
+        u8 rsvd5[5];
+        u32 dsi_ddr_clk;
         u32 bridge_ref_clk;
-        u16 rsvd_pwr;
 
-        /* Dphy Params */
-        u32 prepare_cnt:5;
-        u32 rsvd6:3;
+#define  BYTE_CLK_SEL_20MHZ             0
+#define  BYTE_CLK_SEL_10MHZ             1
+#define  BYTE_CLK_SEL_5MHZ              2
+        u8 byte_clk_sel:2;
+
+        u8 rsvd6:6;
+
+        /* DPHY Flags */
+        u16 dphy_param_valid:1;
+        u16 eot_pkt_disabled:1;
+        u16 enable_clk_stop:1;
+        u16 rsvd7:13;
+
+        u32 hs_tx_timeout;
+        u32 lp_rx_timeout;
+        u32 turn_around_timeout;
+        u32 device_reset_timer;
+        u32 master_init_timer;
+        u32 dbi_bw_timer;
+        u32 lp_byte_clk_val;
+
+        /*  4 byte Dphy Params */
+        u32 prepare_cnt:6;
+        u32 rsvd8:2;
         u32 clk_zero_cnt:8;
         u32 trail_cnt:5;
-        u32 rsvd7:3;
+        u32 rsvd9:3;
         u32 exit_zero_cnt:6;
-        u32 rsvd8:2;
+        u32 rsvd10:2;
 
-        u32 hl_switch_cnt;
-        u32 lp_byte_clk;
         u32 clk_lane_switch_cnt;
+        u32 hl_switch_cnt;
+
+        u32 rsvd11[6];
+
+        /* timings based on dphy spec */
+        u8 tclk_miss;
+        u8 tclk_post;
+        u8 rsvd12;
+        u8 tclk_pre;
+        u8 tclk_prepare;
+        u8 tclk_settle;
+        u8 tclk_term_enable;
+        u8 tclk_trail;
+        u16 tclk_prepare_clkzero;
+        u8 rsvd13;
+        u8 td_term_enable;
+        u8 teot;
+        u8 ths_exit;
+        u8 ths_prepare;
+        u16 ths_prepare_hszero;
+        u8 rsvd14;
+        u8 ths_settle;
+        u8 ths_skip;
+        u8 ths_trail;
+        u8 tinit;
+        u8 tlpx;
+        u8 rsvd15[3];
+
+        /* GPIOs */
+        u8 panel_enable;
+        u8 bl_enable;
+        u8 pwm_enable;
+        u8 reset_r_n;
+        u8 pwr_down_r;
+        u8 stdby_r_n;
+
 } __packed;
 
+/* Block 52 contains MIPI configuration block
+ * 6 * bdb_mipi_config, followed by 6 pps data
+ * block below
+ *
+ * all delays has a unit of 100us
+ */
+struct mipi_pps_data {
+        u16 panel_on_delay;
+        u16 bl_enable_delay;
+        u16 bl_disable_delay;
+        u16 panel_off_delay;
+        u16 panel_power_cycle_delay;
+};
+
+struct bdb_mipi_config {
+        struct mipi_config config[MAX_MIPI_CONFIGURATIONS];
+        struct mipi_pps_data pps[MAX_MIPI_CONFIGURATIONS];
+};
+
+/* Block 53 contains MIPI sequences as needed by the panel
+ * for enabling it. This block can be variable in size and
+ * can be maximum of 6 blocks
+ */
+struct bdb_mipi_sequence {
+        u8 version;
+        u8 data[0];
+};
+
 #endif /* _I830_BIOS_H_ */
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index e2e39e65f109..aa5a3dc43342 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -68,8 +68,13 @@ static bool intel_crt_get_hw_state(struct intel_encoder *encoder,
         struct drm_device *dev = encoder->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crt *crt = intel_encoder_to_crt(encoder);
+        enum intel_display_power_domain power_domain;
         u32 tmp;
 
+        power_domain = intel_display_port_power_domain(encoder);
+        if (!intel_display_power_enabled(dev_priv, power_domain))
+                return false;
+
         tmp = I915_READ(crt->adpa_reg);
 
         if (!(tmp & ADPA_DAC_ENABLE))
@@ -262,6 +267,10 @@ static bool intel_crt_compute_config(struct intel_encoder *encoder,
         if (HAS_PCH_LPT(dev))
                 pipe_config->pipe_bpp = 24;
 
+        /* FDI must always be 2.7 GHz */
+        if (HAS_DDI(dev))
+                pipe_config->port_clock = 135000 * 2;
+
         return true;
 }
 
@@ -630,14 +639,22 @@ static enum drm_connector_status
 intel_crt_detect(struct drm_connector *connector, bool force)
 {
         struct drm_device *dev = connector->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crt *crt = intel_attached_crt(connector);
+        struct intel_encoder *intel_encoder = &crt->base;
+        enum intel_display_power_domain power_domain;
         enum drm_connector_status status;
         struct intel_load_detect_pipe tmp;
 
+        intel_runtime_pm_get(dev_priv);
+
         DRM_DEBUG_KMS("[CONNECTOR:%d:%s] force=%d\n",
                       connector->base.id, drm_get_connector_name(connector),
                       force);
 
+        power_domain = intel_display_port_power_domain(intel_encoder);
+        intel_display_power_get(dev_priv, power_domain);
+
         if (I915_HAS_HOTPLUG(dev)) {
                 /* We can not rely on the HPD pin always being correctly wired
                  * up, for example many KVM do not pass it through, and so
@@ -645,23 +662,30 @@ intel_crt_detect(struct drm_connector *connector, bool force)
                  */
                 if (intel_crt_detect_hotplug(connector)) {
                         DRM_DEBUG_KMS("CRT detected via hotplug\n");
-                        return connector_status_connected;
+                        status = connector_status_connected;
+                        goto out;
                 } else
                         DRM_DEBUG_KMS("CRT not detected via hotplug\n");
         }
 
-        if (intel_crt_detect_ddc(connector))
-                return connector_status_connected;
+        if (intel_crt_detect_ddc(connector)) {
+                status = connector_status_connected;
+                goto out;
+        }
 
         /* Load detection is broken on HPD capable machines. Whoever wants a
          * broken monitor (without edid) to work behind a broken kvm (that fails
          * to have the right resistors for HP detection) needs to fix this up.
          * For now just bail out. */
-        if (I915_HAS_HOTPLUG(dev))
-                return connector_status_disconnected;
+        if (I915_HAS_HOTPLUG(dev)) {
+                status = connector_status_disconnected;
+                goto out;
+        }
 
-        if (!force)
-                return connector->status;
+        if (!force) {
+                status = connector->status;
+                goto out;
+        }
 
         /* for pre-945g platforms use load detect */
         if (intel_get_load_detect_pipe(connector, NULL, &tmp)) {
@@ -673,6 +697,10 @@ intel_crt_detect(struct drm_connector *connector, bool force)
         } else
                 status = connector_status_unknown;
 
+out:
+        intel_display_power_put(dev_priv, power_domain);
+        intel_runtime_pm_put(dev_priv);
+
         return status;
 }
 
@@ -686,17 +714,28 @@ static int intel_crt_get_modes(struct drm_connector *connector)
 {
         struct drm_device *dev = connector->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crt *crt = intel_attached_crt(connector);
+        struct intel_encoder *intel_encoder = &crt->base;
+        enum intel_display_power_domain power_domain;
         int ret;
         struct i2c_adapter *i2c;
 
+        power_domain = intel_display_port_power_domain(intel_encoder);
+        intel_display_power_get(dev_priv, power_domain);
+
         i2c = intel_gmbus_get_adapter(dev_priv, dev_priv->vbt.crt_ddc_pin);
         ret = intel_crt_ddc_get_modes(connector, i2c);
         if (ret || !IS_G4X(dev))
-                return ret;
+                goto out;
 
         /* Try to probe digital port for output in DVI-I -> VGA mode. */
         i2c = intel_gmbus_get_adapter(dev_priv, GMBUS_PORT_DPB);
-        return intel_crt_ddc_get_modes(connector, i2c);
+        ret = intel_crt_ddc_get_modes(connector, i2c);
+
+out:
+        intel_display_power_put(dev_priv, power_domain);
+
+        return ret;
 }
 
 static int intel_crt_set_property(struct drm_connector *connector,
@@ -765,6 +804,14 @@ static const struct dmi_system_id intel_no_crt[] = {
                         DMI_MATCH(DMI_PRODUCT_NAME, "ZGB"),
                 },
         },
+        {
+                .callback = intel_no_crt_dmi_callback,
+                .ident = "DELL XPS 8700",
+                .matches = {
+                        DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."),
+                        DMI_MATCH(DMI_PRODUCT_NAME, "XPS 8700"),
+                },
+        },
         { }
 };
 
@@ -800,7 +847,7 @@ void intel_crt_init(struct drm_device *dev)
         intel_connector_attach_encoder(intel_connector, &crt->base);
 
         crt->base.type = INTEL_OUTPUT_ANALOG;
-        crt->base.cloneable = true;
+        crt->base.cloneable = (1 << INTEL_OUTPUT_DVO) | (1 << INTEL_OUTPUT_HDMI);
         if (IS_I830(dev))
                 crt->base.crtc_mask = (1 << 0);
         else
@@ -833,6 +880,7 @@ void intel_crt_init(struct drm_device *dev)
                 crt->base.get_hw_state = intel_crt_get_hw_state;
         }
         intel_connector->get_hw_state = intel_connector_get_hw_state;
+        intel_connector->unregister = intel_connector_unregister;
 
         drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs);
 
@@ -857,4 +905,6 @@ void intel_crt_init(struct drm_device *dev)
 
                 dev_priv->fdi_rx_config = I915_READ(_FDI_RXA_CTL) & fdi_config;
         }
+
+        intel_crt_reset(connector);
 }
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 234ac5f7bc5a..0ad4e9600063 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -633,6 +633,97 @@ static void wrpll_update_rnp(uint64_t freq2k, unsigned budget,
         /* Otherwise a < c && b >= d, do nothing */
 }
 
+static int intel_ddi_calc_wrpll_link(struct drm_i915_private *dev_priv,
+                                     int reg)
+{
+        int refclk = LC_FREQ;
+        int n, p, r;
+        u32 wrpll;
+
+        wrpll = I915_READ(reg);
+        switch (wrpll & SPLL_PLL_REF_MASK) {
+        case SPLL_PLL_SSC:
+        case SPLL_PLL_NON_SSC:
+                /*
+                 * We could calculate spread here, but our checking
+                 * code only cares about 5% accuracy, and spread is a max of
+                 * 0.5% downspread.
+                 */
+                refclk = 135;
+                break;
+        case SPLL_PLL_LCPLL:
+                refclk = LC_FREQ;
+                break;
+        default:
+                WARN(1, "bad wrpll refclk\n");
+                return 0;
+        }
+
+        r = wrpll & WRPLL_DIVIDER_REF_MASK;
+        p = (wrpll & WRPLL_DIVIDER_POST_MASK) >> WRPLL_DIVIDER_POST_SHIFT;
+        n = (wrpll & WRPLL_DIVIDER_FB_MASK) >> WRPLL_DIVIDER_FB_SHIFT;
+
+        /* Convert to KHz, p & r have a fixed point portion */
+        return (refclk * n * 100) / (p * r);
+}
+
+static void intel_ddi_clock_get(struct intel_encoder *encoder,
+                                struct intel_crtc_config *pipe_config)
+{
+        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        enum port port = intel_ddi_get_encoder_port(encoder);
+        int link_clock = 0;
+        u32 val, pll;
+
+        val = I915_READ(PORT_CLK_SEL(port));
+        switch (val & PORT_CLK_SEL_MASK) {
+        case PORT_CLK_SEL_LCPLL_810:
+                link_clock = 81000;
+                break;
+        case PORT_CLK_SEL_LCPLL_1350:
+                link_clock = 135000;
+                break;
+        case PORT_CLK_SEL_LCPLL_2700:
+                link_clock = 270000;
+                break;
+        case PORT_CLK_SEL_WRPLL1:
+                link_clock = intel_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL1);
+                break;
+        case PORT_CLK_SEL_WRPLL2:
+                link_clock = intel_ddi_calc_wrpll_link(dev_priv, WRPLL_CTL2);
+                break;
+        case PORT_CLK_SEL_SPLL:
+                pll = I915_READ(SPLL_CTL) & SPLL_PLL_FREQ_MASK;
+                if (pll == SPLL_PLL_FREQ_810MHz)
+                        link_clock = 81000;
+                else if (pll == SPLL_PLL_FREQ_1350MHz)
+                        link_clock = 135000;
+                else if (pll == SPLL_PLL_FREQ_2700MHz)
+                        link_clock = 270000;
+                else {
+                        WARN(1, "bad spll freq\n");
+                        return;
+                }
+                break;
+        default:
+                WARN(1, "bad port clock sel\n");
+                return;
+        }
+
+        pipe_config->port_clock = link_clock * 2;
+
+        if (pipe_config->has_pch_encoder)
+                pipe_config->adjusted_mode.crtc_clock =
+                        intel_dotclock_calculate(pipe_config->port_clock,
+                                                 &pipe_config->fdi_m_n);
+        else if (pipe_config->has_dp_encoder)
+                pipe_config->adjusted_mode.crtc_clock =
+                        intel_dotclock_calculate(pipe_config->port_clock,
+                                                 &pipe_config->dp_m_n);
+        else
+                pipe_config->adjusted_mode.crtc_clock = pipe_config->port_clock;
+}
+
 static void
 intel_ddi_calculate_wrpll(int clock /* in Hz */,
                           unsigned *r2_out, unsigned *n2_out, unsigned *p_out)
@@ -1017,8 +1108,13 @@ bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector)
         enum port port = intel_ddi_get_encoder_port(intel_encoder);
         enum pipe pipe = 0;
         enum transcoder cpu_transcoder;
+        enum intel_display_power_domain power_domain;
         uint32_t tmp;
 
+        power_domain = intel_display_port_power_domain(intel_encoder);
+        if (!intel_display_power_enabled(dev_priv, power_domain))
+                return false;
+
         if (!intel_encoder->get_hw_state(intel_encoder, &pipe))
                 return false;
 
@@ -1054,9 +1150,14 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder,
         struct drm_device *dev = encoder->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         enum port port = intel_ddi_get_encoder_port(encoder);
+        enum intel_display_power_domain power_domain;
         u32 tmp;
         int i;
 
+        power_domain = intel_display_port_power_domain(encoder);
+        if (!intel_display_power_enabled(dev_priv, power_domain))
+                return false;
+
         tmp = I915_READ(DDI_BUF_CTL(port));
 
         if (!(tmp & DDI_BUF_CTL_ENABLE))
@@ -1200,7 +1301,7 @@ static void intel_ddi_pre_enable(struct intel_encoder *intel_encoder)
 
         if (type == INTEL_OUTPUT_EDP) {
                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
-                ironlake_edp_panel_on(intel_dp);
+                intel_edp_panel_on(intel_dp);
         }
 
         WARN_ON(intel_crtc->ddi_pll_sel == PORT_CLK_SEL_NONE);
@@ -1244,8 +1345,8 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder)
         if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP) {
                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
                 intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
-                ironlake_edp_panel_vdd_on(intel_dp);
-                ironlake_edp_panel_off(intel_dp);
+                intel_edp_panel_vdd_on(intel_dp);
+                intel_edp_panel_off(intel_dp);
         }
 
         I915_WRITE(PORT_CLK_SEL(port), PORT_CLK_SEL_NONE);
@@ -1280,7 +1381,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder)
                 if (port == PORT_A)
                         intel_dp_stop_link_train(intel_dp);
 
-                ironlake_edp_backlight_on(intel_dp);
+                intel_edp_backlight_on(intel_dp);
                 intel_edp_psr_enable(intel_dp);
         }
 
@@ -1313,7 +1414,7 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder)
                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 
                 intel_edp_psr_disable(intel_dp);
-                ironlake_edp_backlight_off(intel_dp);
+                intel_edp_backlight_off(intel_dp);
         }
 }
 
@@ -1325,7 +1426,7 @@ int intel_ddi_get_cdclk_freq(struct drm_i915_private *dev_priv)
 
         if (lcpll & LCPLL_CD_SOURCE_FCLK) {
                 return 800000;
-        } else if (I915_READ(HSW_FUSE_STRAP) & HSW_CDCLK_LIMIT) {
+        } else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT) {
                 return 450000;
         } else if (freq == LCPLL_CLK_FREQ_450) {
                 return 450000;
@@ -1510,6 +1611,8 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
                               pipe_config->pipe_bpp, dev_priv->vbt.edp_bpp);
                 dev_priv->vbt.edp_bpp = pipe_config->pipe_bpp;
         }
+
+        intel_ddi_clock_get(encoder, pipe_config);
 }
 
 static void intel_ddi_destroy(struct drm_encoder *encoder)
@@ -1620,7 +1723,7 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
 
         intel_encoder->type = INTEL_OUTPUT_UNKNOWN;
         intel_encoder->crtc_mask =  (1 << 0) | (1 << 1) | (1 << 2);
-        intel_encoder->cloneable = false;
+        intel_encoder->cloneable = 0;
         intel_encoder->hot_plug = intel_ddi_hot_plug;
 
         if (init_dp)
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 9b8a7c7ea7fc..dae976f51d83 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -51,7 +51,10 @@ static void ironlake_pch_clock_get(struct intel_crtc *crtc,
 
 static int intel_set_mode(struct drm_crtc *crtc, struct drm_display_mode *mode,
                           int x, int y, struct drm_framebuffer *old_fb);
-
+static int intel_framebuffer_init(struct drm_device *dev,
+                                  struct intel_framebuffer *ifb,
+                                  struct drm_mode_fb_cmd2 *mode_cmd,
+                                  struct drm_i915_gem_object *obj);
 
 typedef struct {
         int     min, max;
@@ -738,10 +741,10 @@ bool intel_crtc_active(struct drm_crtc *crtc)
          * We can ditch the adjusted_mode.crtc_clock check as soon
          * as Haswell has gained clock readout/fastboot support.
          *
-         * We can ditch the crtc->fb check as soon as we can
+         * We can ditch the crtc->primary->fb check as soon as we can
          * properly reconstruct framebuffers.
          */
-        return intel_crtc->active && crtc->fb &&
+        return intel_crtc->active && crtc->primary->fb &&
                 intel_crtc->config.adjusted_mode.crtc_clock;
 }
 
@@ -1030,7 +1033,7 @@ static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
         u32 val;
 
         /* ILK FDI PLL is always enabled */
-        if (dev_priv->info->gen == 5)
+        if (INTEL_INFO(dev_priv->dev)->gen == 5)
                 return;
 
         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
@@ -1119,7 +1122,7 @@ void assert_pipe(struct drm_i915_private *dev_priv,
         if (pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE)
                 state = true;
 
-        if (!intel_display_power_enabled(dev_priv->dev,
+        if (!intel_display_power_enabled(dev_priv,
                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder))) {
                 cur_state = false;
         } else {
@@ -1163,7 +1166,7 @@ static void assert_planes_disabled(struct drm_i915_private *dev_priv,
         if (INTEL_INFO(dev)->gen >= 4) {
                 reg = DSPCNTR(pipe);
                 val = I915_READ(reg);
-                WARN((val & DISPLAY_PLANE_ENABLE),
+                WARN(val & DISPLAY_PLANE_ENABLE,
                      "plane %c assertion failure, should be disabled but not\n",
                      plane_name(pipe));
                 return;
@@ -1185,27 +1188,27 @@ static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
                                     enum pipe pipe)
 {
         struct drm_device *dev = dev_priv->dev;
-        int reg, i;
+        int reg, sprite;
         u32 val;
 
         if (IS_VALLEYVIEW(dev)) {
-                for (i = 0; i < dev_priv->num_plane; i++) {
-                        reg = SPCNTR(pipe, i);
+                for_each_sprite(pipe, sprite) {
+                        reg = SPCNTR(pipe, sprite);
                         val = I915_READ(reg);
-                        WARN((val & SP_ENABLE),
+                        WARN(val & SP_ENABLE,
                              "sprite %c assertion failure, should be off on pipe %c but is still active\n",
-                             sprite_name(pipe, i), pipe_name(pipe));
+                             sprite_name(pipe, sprite), pipe_name(pipe));
                 }
         } else if (INTEL_INFO(dev)->gen >= 7) {
                 reg = SPRCTL(pipe);
                 val = I915_READ(reg);
-                WARN((val & SPRITE_ENABLE),
+                WARN(val & SPRITE_ENABLE,
                      "sprite %c assertion failure, should be off on pipe %c but is still active\n",
                      plane_name(pipe), pipe_name(pipe));
         } else if (INTEL_INFO(dev)->gen >= 5) {
                 reg = DVSCNTR(pipe);
                 val = I915_READ(reg);
-                WARN((val & DVS_ENABLE),
+                WARN(val & DVS_ENABLE,
                      "sprite %c assertion failure, should be off on pipe %c but is still active\n",
                      plane_name(pipe), pipe_name(pipe));
         }
@@ -1443,7 +1446,7 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
         assert_pipe_disabled(dev_priv, crtc->pipe);
 
         /* No really, not for ILK+ */
-        BUG_ON(dev_priv->info->gen >= 5);
+        BUG_ON(INTEL_INFO(dev)->gen >= 5);
 
         /* PLL is protected by panel, make sure we can write it */
         if (IS_MOBILE(dev) && !IS_I830(dev))
@@ -1549,11 +1552,12 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
  */
 static void ironlake_enable_shared_dpll(struct intel_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
 
         /* PCH PLLs only available on ILK, SNB and IVB */
-        BUG_ON(dev_priv->info->gen < 5);
+        BUG_ON(INTEL_INFO(dev)->gen < 5);
         if (WARN_ON(pll == NULL))
                 return;
 
@@ -1578,11 +1582,12 @@ static void ironlake_enable_shared_dpll(struct intel_crtc *crtc)
 
 static void intel_disable_shared_dpll(struct intel_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
 
         /* PCH only available on ILK+ */
-        BUG_ON(dev_priv->info->gen < 5);
+        BUG_ON(INTEL_INFO(dev)->gen < 5);
         if (WARN_ON(pll == NULL))
                return;
 
@@ -1617,7 +1622,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
         uint32_t reg, val, pipeconf_val;
 
         /* PCH only available on ILK+ */
-        BUG_ON(dev_priv->info->gen < 5);
+        BUG_ON(INTEL_INFO(dev)->gen < 5);
 
         /* Make sure PCH DPLL is enabled */
         assert_shared_dpll_enabled(dev_priv,
@@ -1670,7 +1675,7 @@ static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
         u32 val, pipeconf_val;
 
         /* PCH only available on ILK+ */
-        BUG_ON(dev_priv->info->gen < 5);
+        BUG_ON(INTEL_INFO(dev_priv->dev)->gen < 5);
 
         /* FDI must be feeding us bits for PCH ports */
         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
@@ -1744,21 +1749,16 @@ static void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
 
 /**
  * intel_enable_pipe - enable a pipe, asserting requirements
- * @dev_priv: i915 private structure
- * @pipe: pipe to enable
- * @pch_port: on ILK+, is this pipe driving a PCH port or not
+ * @crtc: crtc responsible for the pipe
  *
- * Enable @pipe, making sure that various hardware specific requirements
+ * Enable @crtc's pipe, making sure that various hardware specific requirements
  * are met, if applicable, e.g. PLL enabled, LVDS pairs enabled, etc.
- *
- * @pipe should be %PIPE_A or %PIPE_B.
- *
- * Will wait until the pipe is actually running (i.e. first vblank) before
- * returning.
  */
-static void intel_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe,
-                              bool pch_port, bool dsi)
+static void intel_enable_pipe(struct intel_crtc *crtc)
 {
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum pipe pipe = crtc->pipe;
         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
                                                                       pipe);
         enum pipe pch_transcoder;
@@ -1780,12 +1780,12 @@ static void intel_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe,
          * need the check.
          */
         if (!HAS_PCH_SPLIT(dev_priv->dev))
-                if (dsi)
+                if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DSI))
                         assert_dsi_pll_enabled(dev_priv);
                 else
                         assert_pll_enabled(dev_priv, pipe);
         else {
-                if (pch_port) {
+                if (crtc->config.has_pch_encoder) {
                         /* if driving the PCH, we need FDI enabled */
                         assert_fdi_rx_pll_enabled(dev_priv, pch_transcoder);
                         assert_fdi_tx_pll_enabled(dev_priv,
@@ -1796,11 +1796,24 @@ static void intel_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe,
 
         reg = PIPECONF(cpu_transcoder);
         val = I915_READ(reg);
-        if (val & PIPECONF_ENABLE)
+        if (val & PIPECONF_ENABLE) {
+                WARN_ON(!(pipe == PIPE_A &&
+                          dev_priv->quirks & QUIRK_PIPEA_FORCE));
                 return;
+        }
 
         I915_WRITE(reg, val | PIPECONF_ENABLE);
-        intel_wait_for_vblank(dev_priv->dev, pipe);
+        POSTING_READ(reg);
+
+        /*
+         * There's no guarantee the pipe will really start running now. It
+         * depends on the Gen, the output type and the relative order between
+         * pipe and plane enabling. Avoid waiting on HSW+ since it's not
+         * necessary.
+         * TODO: audit the previous gens.
+         */
+        if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
+                intel_wait_for_vblank(dev_priv->dev, pipe);
 }
 
 /**
@@ -1851,22 +1864,23 @@ static void intel_disable_pipe(struct drm_i915_private *dev_priv,
 void intel_flush_primary_plane(struct drm_i915_private *dev_priv,
                                enum plane plane)
 {
-        u32 reg = dev_priv->info->gen >= 4 ? DSPSURF(plane) : DSPADDR(plane);
+        struct drm_device *dev = dev_priv->dev;
+        u32 reg = INTEL_INFO(dev)->gen >= 4 ? DSPSURF(plane) : DSPADDR(plane);
 
         I915_WRITE(reg, I915_READ(reg));
         POSTING_READ(reg);
 }
 
 /**
- * intel_enable_primary_plane - enable the primary plane on a given pipe
+ * intel_enable_primary_hw_plane - enable the primary plane on a given pipe
  * @dev_priv: i915 private structure
  * @plane: plane to enable
  * @pipe: pipe being fed
  *
  * Enable @plane on @pipe, making sure that @pipe is running first.
  */
-static void intel_enable_primary_plane(struct drm_i915_private *dev_priv,
-                                       enum plane plane, enum pipe pipe)
+static void intel_enable_primary_hw_plane(struct drm_i915_private *dev_priv,
+                                          enum plane plane, enum pipe pipe)
 {
         struct intel_crtc *intel_crtc =
                 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
@@ -1891,15 +1905,15 @@ static void intel_enable_primary_plane(struct drm_i915_private *dev_priv,
 }
 
 /**
- * intel_disable_primary_plane - disable the primary plane
+ * intel_disable_primary_hw_plane - disable the primary hardware plane
  * @dev_priv: i915 private structure
  * @plane: plane to disable
  * @pipe: pipe consuming the data
  *
  * Disable @plane; should be an independent operation.
  */
-static void intel_disable_primary_plane(struct drm_i915_private *dev_priv,
-                                        enum plane plane, enum pipe pipe)
+static void intel_disable_primary_hw_plane(struct drm_i915_private *dev_priv,
+                                           enum plane plane, enum pipe pipe)
 {
         struct intel_crtc *intel_crtc =
                 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
@@ -1929,6 +1943,14 @@ static bool need_vtd_wa(struct drm_device *dev)
         return false;
 }
 
+static int intel_align_height(struct drm_device *dev, int height, bool tiled)
+{
+        int tile_height;
+
+        tile_height = tiled ? (IS_GEN2(dev) ? 16 : 8) : 1;
+        return ALIGN(height, tile_height);
+}
+
 int
 intel_pin_and_fence_fb_obj(struct drm_device *dev,
                            struct drm_i915_gem_object *obj,
@@ -2025,8 +2047,114 @@ unsigned long intel_gen4_compute_page_offset(int *x, int *y,
         }
 }
 
-static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb,
-                             int x, int y)
+int intel_format_to_fourcc(int format)
+{
+        switch (format) {
+        case DISPPLANE_8BPP:
+                return DRM_FORMAT_C8;
+        case DISPPLANE_BGRX555:
+                return DRM_FORMAT_XRGB1555;
+        case DISPPLANE_BGRX565:
+                return DRM_FORMAT_RGB565;
+        default:
+        case DISPPLANE_BGRX888:
+                return DRM_FORMAT_XRGB8888;
+        case DISPPLANE_RGBX888:
+                return DRM_FORMAT_XBGR8888;
+        case DISPPLANE_BGRX101010:
+                return DRM_FORMAT_XRGB2101010;
+        case DISPPLANE_RGBX101010:
+                return DRM_FORMAT_XBGR2101010;
+        }
+}
+
+static bool intel_alloc_plane_obj(struct intel_crtc *crtc,
+                                  struct intel_plane_config *plane_config)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_gem_object *obj = NULL;
+        struct drm_mode_fb_cmd2 mode_cmd = { 0 };
+        u32 base = plane_config->base;
+
+        if (plane_config->size == 0)
+                return false;
+
+        obj = i915_gem_object_create_stolen_for_preallocated(dev, base, base,
+                                                             plane_config->size);
+        if (!obj)
+                return false;
+
+        if (plane_config->tiled) {
+                obj->tiling_mode = I915_TILING_X;
+                obj->stride = crtc->base.primary->fb->pitches[0];
+        }
+
+        mode_cmd.pixel_format = crtc->base.primary->fb->pixel_format;
+        mode_cmd.width = crtc->base.primary->fb->width;
+        mode_cmd.height = crtc->base.primary->fb->height;
+        mode_cmd.pitches[0] = crtc->base.primary->fb->pitches[0];
+
+        mutex_lock(&dev->struct_mutex);
+
+        if (intel_framebuffer_init(dev, to_intel_framebuffer(crtc->base.primary->fb),
+                                   &mode_cmd, obj)) {
+                DRM_DEBUG_KMS("intel fb init failed\n");
+                goto out_unref_obj;
+        }
+
+        mutex_unlock(&dev->struct_mutex);
+
+        DRM_DEBUG_KMS("plane fb obj %p\n", obj);
+        return true;
+
+out_unref_obj:
+        drm_gem_object_unreference(&obj->base);
+        mutex_unlock(&dev->struct_mutex);
+        return false;
+}
+
+static void intel_find_plane_obj(struct intel_crtc *intel_crtc,
+                                 struct intel_plane_config *plane_config)
+{
+        struct drm_device *dev = intel_crtc->base.dev;
+        struct drm_crtc *c;
+        struct intel_crtc *i;
+        struct intel_framebuffer *fb;
+
+        if (!intel_crtc->base.primary->fb)
+                return;
+
+        if (intel_alloc_plane_obj(intel_crtc, plane_config))
+                return;
+
+        kfree(intel_crtc->base.primary->fb);
+        intel_crtc->base.primary->fb = NULL;
+
+        /*
+         * Failed to alloc the obj, check to see if we should share
+         * an fb with another CRTC instead
+         */
+        list_for_each_entry(c, &dev->mode_config.crtc_list, head) {
+                i = to_intel_crtc(c);
+
+                if (c == &intel_crtc->base)
+                        continue;
+
+                if (!i->active || !c->primary->fb)
+                        continue;
+
+                fb = to_intel_framebuffer(c->primary->fb);
+                if (i915_gem_obj_ggtt_offset(fb->obj) == plane_config->base) {
+                        drm_framebuffer_reference(c->primary->fb);
+                        intel_crtc->base.primary->fb = c->primary->fb;
+                        break;
+                }
+        }
+}
+
+static int i9xx_update_primary_plane(struct drm_crtc *crtc,
+                                     struct drm_framebuffer *fb,
+                                     int x, int y)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -2125,8 +2253,9 @@ static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         return 0;
 }
 
-static int ironlake_update_plane(struct drm_crtc *crtc,
-                                 struct drm_framebuffer *fb, int x, int y)
+static int ironlake_update_primary_plane(struct drm_crtc *crtc,
+                                         struct drm_framebuffer *fb,
+                                         int x, int y)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -2230,7 +2359,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                 dev_priv->display.disable_fbc(dev);
         intel_increase_pllclock(crtc);
 
-        return dev_priv->display.update_plane(crtc, fb, x, y);
+        return dev_priv->display.update_primary_plane(crtc, fb, x, y);
 }
 
 void intel_display_handle_reset(struct drm_device *dev)
@@ -2267,11 +2396,13 @@ void intel_display_handle_reset(struct drm_device *dev)
                 /*
                  * FIXME: Once we have proper support for primary planes (and
                  * disabling them without disabling the entire crtc) allow again
-                 * a NULL crtc->fb.
+                 * a NULL crtc->primary->fb.
                  */
-                if (intel_crtc->active && crtc->fb)
-                        dev_priv->display.update_plane(crtc, crtc->fb,
-                                                       crtc->x, crtc->y);
+                if (intel_crtc->active && crtc->primary->fb)
+                        dev_priv->display.update_primary_plane(crtc,
+                                                               crtc->primary->fb,
+                                                               crtc->x,
+                                                               crtc->y);
                 mutex_unlock(&crtc->mutex);
         }
 }
@@ -2299,31 +2430,23 @@ intel_finish_fb(struct drm_framebuffer *old_fb)
         return ret;
 }
 
-static void intel_crtc_update_sarea_pos(struct drm_crtc *crtc, int x, int y)
+static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_master_private *master_priv;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        unsigned long flags;
+        bool pending;
 
-        if (!dev->primary->master)
-                return;
+        if (i915_reset_in_progress(&dev_priv->gpu_error) ||
+            intel_crtc->reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
+                return false;
 
-        master_priv = dev->primary->master->driver_priv;
-        if (!master_priv->sarea_priv)
-                return;
+        spin_lock_irqsave(&dev->event_lock, flags);
+        pending = to_intel_crtc(crtc)->unpin_work != NULL;
+        spin_unlock_irqrestore(&dev->event_lock, flags);
 
-        switch (intel_crtc->pipe) {
-        case 0:
-                master_priv->sarea_priv->pipeA_x = x;
-                master_priv->sarea_priv->pipeA_y = y;
-                break;
-        case 1:
-                master_priv->sarea_priv->pipeB_x = x;
-                master_priv->sarea_priv->pipeB_y = y;
-                break;
-        default:
-                break;
-        }
+        return pending;
 }
 
 static int
@@ -2336,6 +2459,11 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         struct drm_framebuffer *old_fb;
         int ret;
 
+        if (intel_crtc_has_pending_flip(crtc)) {
+                DRM_ERROR("pipe is still busy with an old pageflip\n");
+                return -EBUSY;
+        }
+
         /* no fb bound */
         if (!fb) {
                 DRM_ERROR("No FB bound\n");
@@ -2353,8 +2481,8 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         ret = intel_pin_and_fence_fb_obj(dev,
                                          to_intel_framebuffer(fb)->obj,
                                          NULL);
+        mutex_unlock(&dev->struct_mutex);
         if (ret != 0) {
-                mutex_unlock(&dev->struct_mutex);
                 DRM_ERROR("pin & fence failed\n");
                 return ret;
         }
@@ -2372,7 +2500,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
          * whether the platform allows pfit disable with pipe active, and only
          * then update the pipesrc and pfit state, even on the flip path.
          */
-        if (i915_fastboot) {
+        if (i915.fastboot) {
                 const struct drm_display_mode *adjusted_mode =
                         &intel_crtc->config.adjusted_mode;
 
@@ -2390,31 +2518,33 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
                 intel_crtc->config.pipe_src_h = adjusted_mode->crtc_vdisplay;
         }
 
-        ret = dev_priv->display.update_plane(crtc, fb, x, y);
+        ret = dev_priv->display.update_primary_plane(crtc, fb, x, y);
         if (ret) {
+                mutex_lock(&dev->struct_mutex);
                 intel_unpin_fb_obj(to_intel_framebuffer(fb)->obj);
                 mutex_unlock(&dev->struct_mutex);
                 DRM_ERROR("failed to update base address\n");
                 return ret;
         }
 
-        old_fb = crtc->fb;
-        crtc->fb = fb;
+        old_fb = crtc->primary->fb;
+        crtc->primary->fb = fb;
         crtc->x = x;
         crtc->y = y;
 
         if (old_fb) {
                 if (intel_crtc->active && old_fb != fb)
                         intel_wait_for_vblank(dev, intel_crtc->pipe);
+                mutex_lock(&dev->struct_mutex);
                 intel_unpin_fb_obj(to_intel_framebuffer(old_fb)->obj);
+                mutex_unlock(&dev->struct_mutex);
         }
 
+        mutex_lock(&dev->struct_mutex);
         intel_update_fbc(dev);
         intel_edp_psr_update(dev);
         mutex_unlock(&dev->struct_mutex);
 
-        intel_crtc_update_sarea_pos(crtc, x, y);
-
         return 0;
 }
 
@@ -2963,25 +3093,6 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc)
         udelay(100);
 }
 
-static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc)
-{
-        struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        unsigned long flags;
-        bool pending;
-
-        if (i915_reset_in_progress(&dev_priv->gpu_error) ||
-            intel_crtc->reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
-                return false;
-
-        spin_lock_irqsave(&dev->event_lock, flags);
-        pending = to_intel_crtc(crtc)->unpin_work != NULL;
-        spin_unlock_irqrestore(&dev->event_lock, flags);
-
-        return pending;
-}
-
 bool intel_has_pending_fb_unpin(struct drm_device *dev)
 {
         struct intel_crtc *crtc;
@@ -3011,7 +3122,7 @@ static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        if (crtc->fb == NULL)
+        if (crtc->primary->fb == NULL)
                 return;
 
         WARN_ON(waitqueue_active(&dev_priv->pending_flip_queue));
@@ -3020,7 +3131,7 @@ static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
                    !intel_crtc_has_pending_flip(crtc));
 
         mutex_lock(&dev->struct_mutex);
-        intel_finish_fb(crtc->fb);
+        intel_finish_fb(crtc->primary->fb);
         mutex_unlock(&dev->struct_mutex);
 }
 
@@ -3425,22 +3536,28 @@ static void intel_enable_planes(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
         enum pipe pipe = to_intel_crtc(crtc)->pipe;
+        struct drm_plane *plane;
         struct intel_plane *intel_plane;
 
-        list_for_each_entry(intel_plane, &dev->mode_config.plane_list, base.head)
+        drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) {
+                intel_plane = to_intel_plane(plane);
                 if (intel_plane->pipe == pipe)
                         intel_plane_restore(&intel_plane->base);
+        }
 }
 
 static void intel_disable_planes(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
         enum pipe pipe = to_intel_crtc(crtc)->pipe;
+        struct drm_plane *plane;
         struct intel_plane *intel_plane;
 
-        list_for_each_entry(intel_plane, &dev->mode_config.plane_list, base.head)
+        drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) {
+                intel_plane = to_intel_plane(plane);
                 if (intel_plane->pipe == pipe)
                         intel_plane_disable(&intel_plane->base);
+        }
 }
 
 void hsw_enable_ips(struct intel_crtc *crtc)
@@ -3587,9 +3704,8 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
         intel_crtc_load_lut(crtc);
 
         intel_update_watermarks(crtc);
-        intel_enable_pipe(dev_priv, pipe,
-                          intel_crtc->config.has_pch_encoder, false);
-        intel_enable_primary_plane(dev_priv, plane, pipe);
+        intel_enable_pipe(intel_crtc);
+        intel_enable_primary_hw_plane(dev_priv, plane, pipe);
         intel_enable_planes(crtc);
         intel_crtc_update_cursor(crtc, true);
 
@@ -3631,7 +3747,7 @@ static void haswell_crtc_enable_planes(struct drm_crtc *crtc)
         int pipe = intel_crtc->pipe;
         int plane = intel_crtc->plane;
 
-        intel_enable_primary_plane(dev_priv, plane, pipe);
+        intel_enable_primary_hw_plane(dev_priv, plane, pipe);
         intel_enable_planes(crtc);
         intel_crtc_update_cursor(crtc, true);
 
@@ -3661,7 +3777,7 @@ static void haswell_crtc_disable_planes(struct drm_crtc *crtc)
 
         intel_crtc_update_cursor(crtc, false);
         intel_disable_planes(crtc);
-        intel_disable_primary_plane(dev_priv, plane, pipe);
+        intel_disable_primary_hw_plane(dev_priv, plane, pipe);
 }
 
 /*
@@ -3733,8 +3849,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
         intel_ddi_enable_transcoder_func(crtc);
 
         intel_update_watermarks(crtc);
-        intel_enable_pipe(dev_priv, pipe,
-                          intel_crtc->config.has_pch_encoder, false);
+        intel_enable_pipe(intel_crtc);
 
         if (intel_crtc->config.has_pch_encoder)
                 lpt_pch_enable(crtc);
@@ -3748,16 +3863,6 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
          * to change the workaround. */
         haswell_mode_set_planes_workaround(intel_crtc);
         haswell_crtc_enable_planes(crtc);
-
-        /*
-         * There seems to be a race in PCH platform hw (at least on some
-         * outputs) where an enabled pipe still completes any pageflip right
-         * away (as if the pipe is off) instead of waiting for vblank. As soon
-         * as the first vblank happend, everything works as expected. Hence just
-         * wait for one vblank before returning to avoid strange things
-         * happening.
-         */
-        intel_wait_for_vblank(dev, intel_crtc->pipe);
 }
 
 static void ironlake_pfit_disable(struct intel_crtc *crtc)
@@ -3800,7 +3905,7 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 
         intel_crtc_update_cursor(crtc, false);
         intel_disable_planes(crtc);
-        intel_disable_primary_plane(dev_priv, plane, pipe);
+        intel_disable_primary_hw_plane(dev_priv, plane, pipe);
 
         if (intel_crtc->config.has_pch_encoder)
                 intel_set_pch_fifo_underrun_reporting(dev, pipe, false);
@@ -3972,6 +4077,117 @@ static void i9xx_pfit_enable(struct intel_crtc *crtc)
         I915_WRITE(BCLRPAT(crtc->pipe), 0);
 }
 
+#define for_each_power_domain(domain, mask)                             \
+        for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++)     \
+                if ((1 << (domain)) & (mask))
+
+enum intel_display_power_domain
+intel_display_port_power_domain(struct intel_encoder *intel_encoder)
+{
+        struct drm_device *dev = intel_encoder->base.dev;
+        struct intel_digital_port *intel_dig_port;
+
+        switch (intel_encoder->type) {
+        case INTEL_OUTPUT_UNKNOWN:
+                /* Only DDI platforms should ever use this output type */
+                WARN_ON_ONCE(!HAS_DDI(dev));
+        case INTEL_OUTPUT_DISPLAYPORT:
+        case INTEL_OUTPUT_HDMI:
+        case INTEL_OUTPUT_EDP:
+                intel_dig_port = enc_to_dig_port(&intel_encoder->base);
+                switch (intel_dig_port->port) {
+                case PORT_A:
+                        return POWER_DOMAIN_PORT_DDI_A_4_LANES;
+                case PORT_B:
+                        return POWER_DOMAIN_PORT_DDI_B_4_LANES;
+                case PORT_C:
+                        return POWER_DOMAIN_PORT_DDI_C_4_LANES;
+                case PORT_D:
+                        return POWER_DOMAIN_PORT_DDI_D_4_LANES;
+                default:
+                        WARN_ON_ONCE(1);
+                        return POWER_DOMAIN_PORT_OTHER;
+                }
+        case INTEL_OUTPUT_ANALOG:
+                return POWER_DOMAIN_PORT_CRT;
+        case INTEL_OUTPUT_DSI:
+                return POWER_DOMAIN_PORT_DSI;
+        default:
+                return POWER_DOMAIN_PORT_OTHER;
+        }
+}
+
+static unsigned long get_crtc_power_domains(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        struct intel_encoder *intel_encoder;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        enum pipe pipe = intel_crtc->pipe;
+        bool pfit_enabled = intel_crtc->config.pch_pfit.enabled;
+        unsigned long mask;
+        enum transcoder transcoder;
+
+        transcoder = intel_pipe_to_cpu_transcoder(dev->dev_private, pipe);
+
+        mask = BIT(POWER_DOMAIN_PIPE(pipe));
+        mask |= BIT(POWER_DOMAIN_TRANSCODER(transcoder));
+        if (pfit_enabled)
+                mask |= BIT(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
+
+        for_each_encoder_on_crtc(dev, crtc, intel_encoder)
+                mask |= BIT(intel_display_port_power_domain(intel_encoder));
+
+        return mask;
+}
+
+void intel_display_set_init_power(struct drm_i915_private *dev_priv,
+                                  bool enable)
+{
+        if (dev_priv->power_domains.init_power_on == enable)
+                return;
+
+        if (enable)
+                intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
+        else
+                intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
+
+        dev_priv->power_domains.init_power_on = enable;
+}
+
+static void modeset_update_crtc_power_domains(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long pipe_domains[I915_MAX_PIPES] = { 0, };
+        struct intel_crtc *crtc;
+
+        /*
+         * First get all needed power domains, then put all unneeded, to avoid
+         * any unnecessary toggling of the power wells.
+         */
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
+                enum intel_display_power_domain domain;
+
+                if (!crtc->base.enabled)
+                        continue;
+
+                pipe_domains[crtc->pipe] = get_crtc_power_domains(&crtc->base);
+
+                for_each_power_domain(domain, pipe_domains[crtc->pipe])
+                        intel_display_power_get(dev_priv, domain);
+        }
+
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
+                enum intel_display_power_domain domain;
+
+                for_each_power_domain(domain, crtc->enabled_power_domains)
+                        intel_display_power_put(dev_priv, domain);
+
+                crtc->enabled_power_domains = pipe_domains[crtc->pipe];
+        }
+
+        intel_display_set_init_power(dev_priv, false);
+}
+
 int valleyview_get_vco(struct drm_i915_private *dev_priv)
 {
         int hpll_freq, vco_freq[] = { 800, 1600, 2000, 2400 };
@@ -4088,9 +4304,8 @@ static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv,
         /* Looks like the 200MHz CDclk freq doesn't work on some configs */
 }
 
-static int intel_mode_max_pixclk(struct drm_i915_private *dev_priv,
-                                 unsigned modeset_pipes,
-                                 struct intel_crtc_config *pipe_config)
+/* compute the max pixel clock for new configuration */
+static int intel_mode_max_pixclk(struct drm_i915_private *dev_priv)
 {
         struct drm_device *dev = dev_priv->dev;
         struct intel_crtc *intel_crtc;
@@ -4098,31 +4313,26 @@ static int intel_mode_max_pixclk(struct drm_i915_private *dev_priv,
 
         list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
                             base.head) {
-                if (modeset_pipes & (1 << intel_crtc->pipe))
-                        max_pixclk = max(max_pixclk,
-                                         pipe_config->adjusted_mode.crtc_clock);
-                else if (intel_crtc->base.enabled)
+                if (intel_crtc->new_enabled)
                         max_pixclk = max(max_pixclk,
-                                         intel_crtc->config.adjusted_mode.crtc_clock);
+                                         intel_crtc->new_config->adjusted_mode.crtc_clock);
         }
 
         return max_pixclk;
 }
 
 static void valleyview_modeset_global_pipes(struct drm_device *dev,
-                                            unsigned *prepare_pipes,
-                                            unsigned modeset_pipes,
-                                            struct intel_crtc_config *pipe_config)
+                                            unsigned *prepare_pipes)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc;
-        int max_pixclk = intel_mode_max_pixclk(dev_priv, modeset_pipes,
-                                               pipe_config);
+        int max_pixclk = intel_mode_max_pixclk(dev_priv);
         int cur_cdclk = valleyview_cur_cdclk(dev_priv);
 
         if (valleyview_calc_cdclk(dev_priv, max_pixclk) == cur_cdclk)
                 return;
 
+        /* disable/enable all currently active pipes while we change cdclk */
         list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
                             base.head)
                 if (intel_crtc->base.enabled)
@@ -4132,12 +4342,13 @@ static void valleyview_modeset_global_pipes(struct drm_device *dev,
 static void valleyview_modeset_global_resources(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int max_pixclk = intel_mode_max_pixclk(dev_priv, 0, NULL);
+        int max_pixclk = intel_mode_max_pixclk(dev_priv);
         int cur_cdclk = valleyview_cur_cdclk(dev_priv);
         int req_cdclk = valleyview_calc_cdclk(dev_priv, max_pixclk);
 
         if (req_cdclk != cur_cdclk)
                 valleyview_set_cdclk(dev, req_cdclk);
+        modeset_update_crtc_power_domains(dev);
 }
 
 static void valleyview_crtc_enable(struct drm_crtc *crtc)
@@ -4175,8 +4386,9 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
         intel_crtc_load_lut(crtc);
 
         intel_update_watermarks(crtc);
-        intel_enable_pipe(dev_priv, pipe, false, is_dsi);
-        intel_enable_primary_plane(dev_priv, plane, pipe);
+        intel_enable_pipe(intel_crtc);
+        intel_set_cpu_fifo_underrun_reporting(dev, pipe, true);
+        intel_enable_primary_hw_plane(dev_priv, plane, pipe);
         intel_enable_planes(crtc);
         intel_crtc_update_cursor(crtc, true);
 
@@ -4213,8 +4425,9 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc)
         intel_crtc_load_lut(crtc);
 
         intel_update_watermarks(crtc);
-        intel_enable_pipe(dev_priv, pipe, false, false);
-        intel_enable_primary_plane(dev_priv, plane, pipe);
+        intel_enable_pipe(intel_crtc);
+        intel_set_cpu_fifo_underrun_reporting(dev, pipe, true);
+        intel_enable_primary_hw_plane(dev_priv, plane, pipe);
         intel_enable_planes(crtc);
         /* The fixup needs to happen before cursor is enabled */
         if (IS_G4X(dev))
@@ -4270,8 +4483,9 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
         intel_crtc_dpms_overlay(intel_crtc, false);
         intel_crtc_update_cursor(crtc, false);
         intel_disable_planes(crtc);
-        intel_disable_primary_plane(dev_priv, plane, pipe);
+        intel_disable_primary_hw_plane(dev_priv, plane, pipe);
 
+        intel_set_cpu_fifo_underrun_reporting(dev, pipe, false);
         intel_disable_pipe(dev_priv, pipe);
 
         i9xx_pfit_disable(intel_crtc);
@@ -4365,11 +4579,11 @@ static void intel_crtc_disable(struct drm_crtc *crtc)
         assert_cursor_disabled(dev_priv, to_intel_crtc(crtc)->pipe);
         assert_pipe_disabled(dev->dev_private, to_intel_crtc(crtc)->pipe);
 
-        if (crtc->fb) {
+        if (crtc->primary->fb) {
                 mutex_lock(&dev->struct_mutex);
-                intel_unpin_fb_obj(to_intel_framebuffer(crtc->fb)->obj);
+                intel_unpin_fb_obj(to_intel_framebuffer(crtc->primary->fb)->obj);
                 mutex_unlock(&dev->struct_mutex);
-                crtc->fb = NULL;
+                crtc->primary->fb = NULL;
         }
 
         /* Update computed state. */
@@ -4583,7 +4797,7 @@ retry:
 static void hsw_compute_ips_config(struct intel_crtc *crtc,
                                    struct intel_crtc_config *pipe_config)
 {
-        pipe_config->ips_enabled = i915_enable_ips &&
+        pipe_config->ips_enabled = i915.enable_ips &&
                                    hsw_crtc_supports_ips(crtc) &&
                                    pipe_config->pipe_bpp <= 24;
 }
@@ -4784,8 +4998,8 @@ intel_link_compute_m_n(int bits_per_pixel, int nlanes,
 
 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
 {
-        if (i915_panel_use_ssc >= 0)
-                return i915_panel_use_ssc != 0;
+        if (i915.panel_use_ssc >= 0)
+                return i915.panel_use_ssc != 0;
         return dev_priv->vbt.lvds_use_ssc
                 && !(dev_priv->quirks & QUIRK_LVDS_SSC_DISABLE);
 }
@@ -4844,7 +5058,7 @@ static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
 
         crtc->lowfreq_avail = false;
         if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_LVDS) &&
-            reduced_clock && i915_powersave) {
+            reduced_clock && i915.powersave) {
                 I915_WRITE(FP1(pipe), fp2);
                 crtc->config.dpll_hw_state.fp1 = fp2;
                 crtc->lowfreq_avail = true;
@@ -5161,21 +5375,26 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
         enum transcoder cpu_transcoder = intel_crtc->config.cpu_transcoder;
         struct drm_display_mode *adjusted_mode =
                 &intel_crtc->config.adjusted_mode;
-        uint32_t vsyncshift, crtc_vtotal, crtc_vblank_end;
+        uint32_t crtc_vtotal, crtc_vblank_end;
+        int vsyncshift = 0;
 
         /* We need to be careful not to changed the adjusted mode, for otherwise
          * the hw state checker will get angry at the mismatch. */
         crtc_vtotal = adjusted_mode->crtc_vtotal;
         crtc_vblank_end = adjusted_mode->crtc_vblank_end;
 
-        if (!IS_GEN2(dev) && adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
+        if (adjusted_mode->flags & DRM_MODE_FLAG_INTERLACE) {
                 /* the chip adds 2 halflines automatically */
                 crtc_vtotal -= 1;
                 crtc_vblank_end -= 1;
-                vsyncshift = adjusted_mode->crtc_hsync_start
-                             - adjusted_mode->crtc_htotal / 2;
-        } else {
-                vsyncshift = 0;
+
+                if (intel_pipe_has_type(&intel_crtc->base, INTEL_OUTPUT_SDVO))
+                        vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
+                else
+                        vsyncshift = adjusted_mode->crtc_hsync_start -
+                                adjusted_mode->crtc_htotal / 2;
+                if (vsyncshift < 0)
+                        vsyncshift += adjusted_mode->crtc_htotal;
         }
 
         if (INTEL_INFO(dev)->gen > 3)
@@ -5259,25 +5478,23 @@ static void intel_get_pipe_timings(struct intel_crtc *crtc,
         pipe_config->requested_mode.hdisplay = pipe_config->pipe_src_w;
 }
 
-static void intel_crtc_mode_from_pipe_config(struct intel_crtc *intel_crtc,
-                                             struct intel_crtc_config *pipe_config)
+void intel_mode_from_pipe_config(struct drm_display_mode *mode,
+                                 struct intel_crtc_config *pipe_config)
 {
-        struct drm_crtc *crtc = &intel_crtc->base;
-
-        crtc->mode.hdisplay = pipe_config->adjusted_mode.crtc_hdisplay;
-        crtc->mode.htotal = pipe_config->adjusted_mode.crtc_htotal;
-        crtc->mode.hsync_start = pipe_config->adjusted_mode.crtc_hsync_start;
-        crtc->mode.hsync_end = pipe_config->adjusted_mode.crtc_hsync_end;
+        mode->hdisplay = pipe_config->adjusted_mode.crtc_hdisplay;
+        mode->htotal = pipe_config->adjusted_mode.crtc_htotal;
+        mode->hsync_start = pipe_config->adjusted_mode.crtc_hsync_start;
+        mode->hsync_end = pipe_config->adjusted_mode.crtc_hsync_end;
 
-        crtc->mode.vdisplay = pipe_config->adjusted_mode.crtc_vdisplay;
-        crtc->mode.vtotal = pipe_config->adjusted_mode.crtc_vtotal;
-        crtc->mode.vsync_start = pipe_config->adjusted_mode.crtc_vsync_start;
-        crtc->mode.vsync_end = pipe_config->adjusted_mode.crtc_vsync_end;
+        mode->vdisplay = pipe_config->adjusted_mode.crtc_vdisplay;
+        mode->vtotal = pipe_config->adjusted_mode.crtc_vtotal;
+        mode->vsync_start = pipe_config->adjusted_mode.crtc_vsync_start;
+        mode->vsync_end = pipe_config->adjusted_mode.crtc_vsync_end;
 
-        crtc->mode.flags = pipe_config->adjusted_mode.flags;
+        mode->flags = pipe_config->adjusted_mode.flags;
 
-        crtc->mode.clock = pipe_config->adjusted_mode.crtc_clock;
-        crtc->mode.flags |= pipe_config->adjusted_mode.flags;
+        mode->clock = pipe_config->adjusted_mode.crtc_clock;
+        mode->flags |= pipe_config->adjusted_mode.flags;
 }
 
 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
@@ -5327,10 +5544,13 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
                 }
         }
 
-        if (!IS_GEN2(dev) &&
-            intel_crtc->config.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE)
-                pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
-        else
+        if (intel_crtc->config.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
+                if (INTEL_INFO(dev)->gen < 4 ||
+                    intel_pipe_has_type(&intel_crtc->base, INTEL_OUTPUT_SDVO))
+                        pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
+                else
+                        pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
+        } else
                 pipeconf |= PIPECONF_PROGRESSIVE;
 
         if (IS_VALLEYVIEW(dev) && intel_crtc->config.limited_color_range)
@@ -5512,6 +5732,67 @@ static void vlv_crtc_clock_get(struct intel_crtc *crtc,
         pipe_config->port_clock = clock.dot / 5;
 }
 
+static void i9xx_get_plane_config(struct intel_crtc *crtc,
+                                  struct intel_plane_config *plane_config)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 val, base, offset;
+        int pipe = crtc->pipe, plane = crtc->plane;
+        int fourcc, pixel_format;
+        int aligned_height;
+
+        crtc->base.primary->fb = kzalloc(sizeof(struct intel_framebuffer), GFP_KERNEL);
+        if (!crtc->base.primary->fb) {
+                DRM_DEBUG_KMS("failed to alloc fb\n");
+                return;
+        }
+
+        val = I915_READ(DSPCNTR(plane));
+
+        if (INTEL_INFO(dev)->gen >= 4)
+                if (val & DISPPLANE_TILED)
+                        plane_config->tiled = true;
+
+        pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
+        fourcc = intel_format_to_fourcc(pixel_format);
+        crtc->base.primary->fb->pixel_format = fourcc;
+        crtc->base.primary->fb->bits_per_pixel =
+                drm_format_plane_cpp(fourcc, 0) * 8;
+
+        if (INTEL_INFO(dev)->gen >= 4) {
+                if (plane_config->tiled)
+                        offset = I915_READ(DSPTILEOFF(plane));
+                else
+                        offset = I915_READ(DSPLINOFF(plane));
+                base = I915_READ(DSPSURF(plane)) & 0xfffff000;
+        } else {
+                base = I915_READ(DSPADDR(plane));
+        }
+        plane_config->base = base;
+
+        val = I915_READ(PIPESRC(pipe));
+        crtc->base.primary->fb->width = ((val >> 16) & 0xfff) + 1;
+        crtc->base.primary->fb->height = ((val >> 0) & 0xfff) + 1;
+
+        val = I915_READ(DSPSTRIDE(pipe));
+        crtc->base.primary->fb->pitches[0] = val & 0xffffff80;
+
+        aligned_height = intel_align_height(dev, crtc->base.primary->fb->height,
+                                            plane_config->tiled);
+
+        plane_config->size = ALIGN(crtc->base.primary->fb->pitches[0] *
+                                   aligned_height, PAGE_SIZE);
+
+        DRM_DEBUG_KMS("pipe/plane %d/%d with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
+                      pipe, plane, crtc->base.primary->fb->width,
+                      crtc->base.primary->fb->height,
+                      crtc->base.primary->fb->bits_per_pixel, base,
+                      crtc->base.primary->fb->pitches[0],
+                      plane_config->size);
+
+}
+
 static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
                                  struct intel_crtc_config *pipe_config)
 {
@@ -5519,6 +5800,10 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
         struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t tmp;
 
+        if (!intel_display_power_enabled(dev_priv,
+                                         POWER_DOMAIN_PIPE(crtc->pipe)))
+                return false;
+
         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
         pipe_config->shared_dpll = DPLL_ID_PRIVATE;
 
@@ -6180,7 +6465,7 @@ int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp)
          * is 2.5%; use 5% for safety's sake.
          */
         u32 bps = target_clock * bpp * 21 / 20;
-        return bps / (link_bw * 8) + 1;
+        return DIV_ROUND_UP(bps, link_bw * 8);
 }
 
 static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
@@ -6348,7 +6633,7 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
         if (intel_crtc->config.has_dp_encoder)
                 intel_dp_set_m_n(intel_crtc);
 
-        if (is_lvds && has_reduced_clock && i915_powersave)
+        if (is_lvds && has_reduced_clock && i915.powersave)
                 intel_crtc->lowfreq_avail = true;
         else
                 intel_crtc->lowfreq_avail = false;
@@ -6455,6 +6740,66 @@ static void ironlake_get_pfit_config(struct intel_crtc *crtc,
         }
 }
 
+static void ironlake_get_plane_config(struct intel_crtc *crtc,
+                                      struct intel_plane_config *plane_config)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 val, base, offset;
+        int pipe = crtc->pipe, plane = crtc->plane;
+        int fourcc, pixel_format;
+        int aligned_height;
+
+        crtc->base.primary->fb = kzalloc(sizeof(struct intel_framebuffer), GFP_KERNEL);
+        if (!crtc->base.primary->fb) {
+                DRM_DEBUG_KMS("failed to alloc fb\n");
+                return;
+        }
+
+        val = I915_READ(DSPCNTR(plane));
+
+        if (INTEL_INFO(dev)->gen >= 4)
+                if (val & DISPPLANE_TILED)
+                        plane_config->tiled = true;
+
+        pixel_format = val & DISPPLANE_PIXFORMAT_MASK;
+        fourcc = intel_format_to_fourcc(pixel_format);
+        crtc->base.primary->fb->pixel_format = fourcc;
+        crtc->base.primary->fb->bits_per_pixel =
+                drm_format_plane_cpp(fourcc, 0) * 8;
+
+        base = I915_READ(DSPSURF(plane)) & 0xfffff000;
+        if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+                offset = I915_READ(DSPOFFSET(plane));
+        } else {
+                if (plane_config->tiled)
+                        offset = I915_READ(DSPTILEOFF(plane));
+                else
+                        offset = I915_READ(DSPLINOFF(plane));
+        }
+        plane_config->base = base;
+
+        val = I915_READ(PIPESRC(pipe));
+        crtc->base.primary->fb->width = ((val >> 16) & 0xfff) + 1;
+        crtc->base.primary->fb->height = ((val >> 0) & 0xfff) + 1;
+
+        val = I915_READ(DSPSTRIDE(pipe));
+        crtc->base.primary->fb->pitches[0] = val & 0xffffff80;
+
+        aligned_height = intel_align_height(dev, crtc->base.primary->fb->height,
+                                            plane_config->tiled);
+
+        plane_config->size = ALIGN(crtc->base.primary->fb->pitches[0] *
+                                   aligned_height, PAGE_SIZE);
+
+        DRM_DEBUG_KMS("pipe/plane %d/%d with fb: size=%dx%d@%d, offset=%x, pitch %d, size 0x%x\n",
+                      pipe, plane, crtc->base.primary->fb->width,
+                      crtc->base.primary->fb->height,
+                      crtc->base.primary->fb->bits_per_pixel, base,
+                      crtc->base.primary->fb->pitches[0],
+                      plane_config->size);
+}
+
 static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
                                      struct intel_crtc_config *pipe_config)
 {
@@ -6629,6 +6974,7 @@ static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
 static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
 {
         uint32_t val;
+        unsigned long irqflags;
 
         val = I915_READ(LCPLL_CTL);
 
@@ -6636,9 +6982,22 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
                     LCPLL_POWER_DOWN_ALLOW)) == LCPLL_PLL_LOCK)
                 return;
 
-        /* Make sure we're not on PC8 state before disabling PC8, otherwise
-         * we'll hang the machine! */
-        gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
+        /*
+         * Make sure we're not on PC8 state before disabling PC8, otherwise
+         * we'll hang the machine. To prevent PC8 state, just enable force_wake.
+         *
+         * The other problem is that hsw_restore_lcpll() is called as part of
+         * the runtime PM resume sequence, so we can't just call
+         * gen6_gt_force_wake_get() because that function calls
+         * intel_runtime_pm_get(), and we can't change the runtime PM refcount
+         * while we are on the resume sequence. So to solve this problem we have
+         * to call special forcewake code that doesn't touch runtime PM and
+         * doesn't enable the forcewake delayed work.
+         */
+        spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
+        if (dev_priv->uncore.forcewake_count++ == 0)
+                dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL);
+        spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 
         if (val & LCPLL_POWER_DOWN_ALLOW) {
                 val &= ~LCPLL_POWER_DOWN_ALLOW;
@@ -6672,26 +7031,45 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
                         DRM_ERROR("Switching back to LCPLL failed\n");
         }
 
-        gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
+        /* See the big comment above. */
+        spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
+        if (--dev_priv->uncore.forcewake_count == 0)
+                dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
+        spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
 
-void hsw_enable_pc8_work(struct work_struct *__work)
+/*
+ * Package states C8 and deeper are really deep PC states that can only be
+ * reached when all the devices on the system allow it, so even if the graphics
+ * device allows PC8+, it doesn't mean the system will actually get to these
+ * states. Our driver only allows PC8+ when going into runtime PM.
+ *
+ * The requirements for PC8+ are that all the outputs are disabled, the power
+ * well is disabled and most interrupts are disabled, and these are also
+ * requirements for runtime PM. When these conditions are met, we manually do
+ * the other conditions: disable the interrupts, clocks and switch LCPLL refclk
+ * to Fclk. If we're in PC8+ and we get an non-hotplug interrupt, we can hard
+ * hang the machine.
+ *
+ * When we really reach PC8 or deeper states (not just when we allow it) we lose
+ * the state of some registers, so when we come back from PC8+ we need to
+ * restore this state. We don't get into PC8+ if we're not in RC6, so we don't
+ * need to take care of the registers kept by RC6. Notice that this happens even
+ * if we don't put the device in PCI D3 state (which is what currently happens
+ * because of the runtime PM support).
+ *
+ * For more, read "Display Sequences for Package C8" on the hardware
+ * documentation.
+ */
+void hsw_enable_pc8(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv =
-                container_of(to_delayed_work(__work), struct drm_i915_private,
-                             pc8.enable_work);
         struct drm_device *dev = dev_priv->dev;
         uint32_t val;
 
         WARN_ON(!HAS_PC8(dev));
 
-        if (dev_priv->pc8.enabled)
-                return;
-
         DRM_DEBUG_KMS("Enabling package C8+\n");
 
-        dev_priv->pc8.enabled = true;
-
         if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
                 val = I915_READ(SOUTH_DSPCLK_GATE_D);
                 val &= ~PCH_LP_PARTITION_LEVEL_DISABLE;
@@ -6699,51 +7077,21 @@ void hsw_enable_pc8_work(struct work_struct *__work)
         }
 
         lpt_disable_clkout_dp(dev);
-        hsw_pc8_disable_interrupts(dev);
+        hsw_runtime_pm_disable_interrupts(dev);
         hsw_disable_lcpll(dev_priv, true, true);
-
-        intel_runtime_pm_put(dev_priv);
-}
-
-static void __hsw_enable_package_c8(struct drm_i915_private *dev_priv)
-{
-        WARN_ON(!mutex_is_locked(&dev_priv->pc8.lock));
-        WARN(dev_priv->pc8.disable_count < 1,
-             "pc8.disable_count: %d\n", dev_priv->pc8.disable_count);
-
-        dev_priv->pc8.disable_count--;
-        if (dev_priv->pc8.disable_count != 0)
-                return;
-
-        schedule_delayed_work(&dev_priv->pc8.enable_work,
-                              msecs_to_jiffies(i915_pc8_timeout));
 }
 
-static void __hsw_disable_package_c8(struct drm_i915_private *dev_priv)
+void hsw_disable_pc8(struct drm_i915_private *dev_priv)
 {
         struct drm_device *dev = dev_priv->dev;
         uint32_t val;
 
-        WARN_ON(!mutex_is_locked(&dev_priv->pc8.lock));
-        WARN(dev_priv->pc8.disable_count < 0,
-             "pc8.disable_count: %d\n", dev_priv->pc8.disable_count);
-
-        dev_priv->pc8.disable_count++;
-        if (dev_priv->pc8.disable_count != 1)
-                return;
-
         WARN_ON(!HAS_PC8(dev));
 
-        cancel_delayed_work_sync(&dev_priv->pc8.enable_work);
-        if (!dev_priv->pc8.enabled)
-                return;
-
         DRM_DEBUG_KMS("Disabling package C8+\n");
 
-        intel_runtime_pm_get(dev_priv);
-
         hsw_restore_lcpll(dev_priv);
-        hsw_pc8_restore_interrupts(dev);
+        hsw_runtime_pm_restore_interrupts(dev);
         lpt_init_pch_refclk(dev);
 
         if (dev_priv->pch_id == INTEL_PCH_LPT_LP_DEVICE_ID_TYPE) {
@@ -6757,185 +7105,11 @@ static void __hsw_disable_package_c8(struct drm_i915_private *dev_priv)
         mutex_lock(&dev_priv->rps.hw_lock);
         gen6_update_ring_freq(dev);
         mutex_unlock(&dev_priv->rps.hw_lock);
-        dev_priv->pc8.enabled = false;
-}
-
-void hsw_enable_package_c8(struct drm_i915_private *dev_priv)
-{
-        if (!HAS_PC8(dev_priv->dev))
-                return;
-
-        mutex_lock(&dev_priv->pc8.lock);
-        __hsw_enable_package_c8(dev_priv);
-        mutex_unlock(&dev_priv->pc8.lock);
-}
-
-void hsw_disable_package_c8(struct drm_i915_private *dev_priv)
-{
-        if (!HAS_PC8(dev_priv->dev))
-                return;
-
-        mutex_lock(&dev_priv->pc8.lock);
-        __hsw_disable_package_c8(dev_priv);
-        mutex_unlock(&dev_priv->pc8.lock);
-}
-
-static bool hsw_can_enable_package_c8(struct drm_i915_private *dev_priv)
-{
-        struct drm_device *dev = dev_priv->dev;
-        struct intel_crtc *crtc;
-        uint32_t val;
-
-        list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head)
-                if (crtc->base.enabled)
-                        return false;
-
-        /* This case is still possible since we have the i915.disable_power_well
-         * parameter and also the KVMr or something else might be requesting the
-         * power well. */
-        val = I915_READ(HSW_PWR_WELL_DRIVER);
-        if (val != 0) {
-                DRM_DEBUG_KMS("Not enabling PC8: power well on\n");
-                return false;
-        }
-
-        return true;
-}
-
-/* Since we're called from modeset_global_resources there's no way to
- * symmetrically increase and decrease the refcount, so we use
- * dev_priv->pc8.requirements_met to track whether we already have the refcount
- * or not.
- */
-static void hsw_update_package_c8(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        bool allow;
-
-        if (!HAS_PC8(dev_priv->dev))
-                return;
-
-        if (!i915_enable_pc8)
-                return;
-
-        mutex_lock(&dev_priv->pc8.lock);
-
-        allow = hsw_can_enable_package_c8(dev_priv);
-
-        if (allow == dev_priv->pc8.requirements_met)
-                goto done;
-
-        dev_priv->pc8.requirements_met = allow;
-
-        if (allow)
-                __hsw_enable_package_c8(dev_priv);
-        else
-                __hsw_disable_package_c8(dev_priv);
-
-done:
-        mutex_unlock(&dev_priv->pc8.lock);
-}
-
-static void hsw_package_c8_gpu_idle(struct drm_i915_private *dev_priv)
-{
-        if (!HAS_PC8(dev_priv->dev))
-                return;
-
-        mutex_lock(&dev_priv->pc8.lock);
-        if (!dev_priv->pc8.gpu_idle) {
-                dev_priv->pc8.gpu_idle = true;
-                __hsw_enable_package_c8(dev_priv);
-        }
-        mutex_unlock(&dev_priv->pc8.lock);
-}
-
-static void hsw_package_c8_gpu_busy(struct drm_i915_private *dev_priv)
-{
-        if (!HAS_PC8(dev_priv->dev))
-                return;
-
-        mutex_lock(&dev_priv->pc8.lock);
-        if (dev_priv->pc8.gpu_idle) {
-                dev_priv->pc8.gpu_idle = false;
-                __hsw_disable_package_c8(dev_priv);
-        }
-        mutex_unlock(&dev_priv->pc8.lock);
-}
-
-#define for_each_power_domain(domain, mask)                             \
-        for ((domain) = 0; (domain) < POWER_DOMAIN_NUM; (domain)++)     \
-                if ((1 << (domain)) & (mask))
-
-static unsigned long get_pipe_power_domains(struct drm_device *dev,
-                                            enum pipe pipe, bool pfit_enabled)
-{
-        unsigned long mask;
-        enum transcoder transcoder;
-
-        transcoder = intel_pipe_to_cpu_transcoder(dev->dev_private, pipe);
-
-        mask = BIT(POWER_DOMAIN_PIPE(pipe));
-        mask |= BIT(POWER_DOMAIN_TRANSCODER(transcoder));
-        if (pfit_enabled)
-                mask |= BIT(POWER_DOMAIN_PIPE_PANEL_FITTER(pipe));
-
-        return mask;
-}
-
-void intel_display_set_init_power(struct drm_device *dev, bool enable)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        if (dev_priv->power_domains.init_power_on == enable)
-                return;
-
-        if (enable)
-                intel_display_power_get(dev, POWER_DOMAIN_INIT);
-        else
-                intel_display_power_put(dev, POWER_DOMAIN_INIT);
-
-        dev_priv->power_domains.init_power_on = enable;
-}
-
-static void modeset_update_power_wells(struct drm_device *dev)
-{
-        unsigned long pipe_domains[I915_MAX_PIPES] = { 0, };
-        struct intel_crtc *crtc;
-
-        /*
-         * First get all needed power domains, then put all unneeded, to avoid
-         * any unnecessary toggling of the power wells.
-         */
-        list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
-                enum intel_display_power_domain domain;
-
-                if (!crtc->base.enabled)
-                        continue;
-
-                pipe_domains[crtc->pipe] = get_pipe_power_domains(dev,
-                                                crtc->pipe,
-                                                crtc->config.pch_pfit.enabled);
-
-                for_each_power_domain(domain, pipe_domains[crtc->pipe])
-                        intel_display_power_get(dev, domain);
-        }
-
-        list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
-                enum intel_display_power_domain domain;
-
-                for_each_power_domain(domain, crtc->enabled_power_domains)
-                        intel_display_power_put(dev, domain);
-
-                crtc->enabled_power_domains = pipe_domains[crtc->pipe];
-        }
-
-        intel_display_set_init_power(dev, false);
 }
 
 static void haswell_modeset_global_resources(struct drm_device *dev)
 {
-        modeset_update_power_wells(dev);
-        hsw_update_package_c8(dev);
+        modeset_update_crtc_power_domains(dev);
 }
 
 static int haswell_crtc_mode_set(struct drm_crtc *crtc,
@@ -6985,6 +7159,10 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
         enum intel_display_power_domain pfit_domain;
         uint32_t tmp;
 
+        if (!intel_display_power_enabled(dev_priv,
+                                         POWER_DOMAIN_PIPE(crtc->pipe)))
+                return false;
+
         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
         pipe_config->shared_dpll = DPLL_ID_PRIVATE;
 
@@ -7010,7 +7188,7 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
                         pipe_config->cpu_transcoder = TRANSCODER_EDP;
         }
 
-        if (!intel_display_power_enabled(dev,
+        if (!intel_display_power_enabled(dev_priv,
                         POWER_DOMAIN_TRANSCODER(pipe_config->cpu_transcoder)))
                 return false;
 
@@ -7038,7 +7216,7 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
         intel_get_pipe_timings(crtc, pipe_config);
 
         pfit_domain = POWER_DOMAIN_PIPE_PANEL_FITTER(crtc->pipe);
-        if (intel_display_power_enabled(dev, pfit_domain))
+        if (intel_display_power_enabled(dev_priv, pfit_domain))
                 ironlake_get_pfit_config(crtc, pipe_config);
 
         if (IS_HASWELL(dev))
@@ -7435,10 +7613,26 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base)
         bool visible = base != 0;
 
         if (intel_crtc->cursor_visible != visible) {
+                int16_t width = intel_crtc->cursor_width;
                 uint32_t cntl = I915_READ(CURCNTR(pipe));
                 if (base) {
                         cntl &= ~(CURSOR_MODE | MCURSOR_PIPE_SELECT);
-                        cntl |= CURSOR_MODE_64_ARGB_AX | MCURSOR_GAMMA_ENABLE;
+                        cntl |= MCURSOR_GAMMA_ENABLE;
+
+                        switch (width) {
+                        case 64:
+                                cntl |= CURSOR_MODE_64_ARGB_AX;
+                                break;
+                        case 128:
+                                cntl |= CURSOR_MODE_128_ARGB_AX;
+                                break;
+                        case 256:
+                                cntl |= CURSOR_MODE_256_ARGB_AX;
+                                break;
+                        default:
+                                WARN_ON(1);
+                                return;
+                        }
                         cntl |= pipe << 28; /* Connect to correct pipe */
                 } else {
                         cntl &= ~(CURSOR_MODE | MCURSOR_GAMMA_ENABLE);
@@ -7463,10 +7657,25 @@ static void ivb_update_cursor(struct drm_crtc *crtc, u32 base)
         bool visible = base != 0;
 
         if (intel_crtc->cursor_visible != visible) {
+                int16_t width = intel_crtc->cursor_width;
                 uint32_t cntl = I915_READ(CURCNTR_IVB(pipe));
                 if (base) {
                         cntl &= ~CURSOR_MODE;
-                        cntl |= CURSOR_MODE_64_ARGB_AX | MCURSOR_GAMMA_ENABLE;
+                        cntl |= MCURSOR_GAMMA_ENABLE;
+                        switch (width) {
+                        case 64:
+                                cntl |= CURSOR_MODE_64_ARGB_AX;
+                                break;
+                        case 128:
+                                cntl |= CURSOR_MODE_128_ARGB_AX;
+                                break;
+                        case 256:
+                                cntl |= CURSOR_MODE_256_ARGB_AX;
+                                break;
+                        default:
+                                WARN_ON(1);
+                                return;
+                        }
                 } else {
                         cntl &= ~(CURSOR_MODE | MCURSOR_GAMMA_ENABLE);
                         cntl |= CURSOR_MODE_DISABLE;
@@ -7550,6 +7759,7 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct drm_i915_gem_object *obj;
+        unsigned old_width;
         uint32_t addr;
         int ret;
 
@@ -7562,9 +7772,11 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
                 goto finish;
         }
 
-        /* Currently we only support 64x64 cursors */
-        if (width != 64 || height != 64) {
-                DRM_ERROR("we currently only support 64x64 cursors\n");
+        /* Check for which cursor types we support */
+        if (!((width == 64 && height == 64) ||
+                        (width == 128 && height == 128 && !IS_GEN2(dev)) ||
+                        (width == 256 && height == 256 && !IS_GEN2(dev)))) {
+                DRM_DEBUG("Cursor dimension not supported\n");
                 return -EINVAL;
         }
 
@@ -7573,18 +7785,18 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
                 return -ENOENT;
 
         if (obj->base.size < width * height * 4) {
-                DRM_ERROR("buffer is to small\n");
+                DRM_DEBUG_KMS("buffer is to small\n");
                 ret = -ENOMEM;
                 goto fail;
         }
 
         /* we only need to pin inside GTT if cursor is non-phy */
         mutex_lock(&dev->struct_mutex);
-        if (!dev_priv->info->cursor_needs_physical) {
+        if (!INTEL_INFO(dev)->cursor_needs_physical) {
                 unsigned alignment;
 
                 if (obj->tiling_mode) {
-                        DRM_ERROR("cursor cannot be tiled\n");
+                        DRM_DEBUG_KMS("cursor cannot be tiled\n");
                         ret = -EINVAL;
                         goto fail_locked;
                 }
@@ -7600,13 +7812,13 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
 
                 ret = i915_gem_object_pin_to_display_plane(obj, alignment, NULL);
                 if (ret) {
-                        DRM_ERROR("failed to move cursor bo into the GTT\n");
+                        DRM_DEBUG_KMS("failed to move cursor bo into the GTT\n");
                         goto fail_locked;
                 }
 
                 ret = i915_gem_object_put_fence(obj);
                 if (ret) {
-                        DRM_ERROR("failed to release fence for cursor");
+                        DRM_DEBUG_KMS("failed to release fence for cursor");
                         goto fail_unpin;
                 }
 
@@ -7617,7 +7829,7 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
                                                   (intel_crtc->pipe == 0) ? I915_GEM_PHYS_CURSOR_0 : I915_GEM_PHYS_CURSOR_1,
                                                   align);
                 if (ret) {
-                        DRM_ERROR("failed to attach phys object\n");
+                        DRM_DEBUG_KMS("failed to attach phys object\n");
                         goto fail_locked;
                 }
                 addr = obj->phys_obj->handle->busaddr;
@@ -7628,7 +7840,7 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
 
  finish:
         if (intel_crtc->cursor_bo) {
-                if (dev_priv->info->cursor_needs_physical) {
+                if (INTEL_INFO(dev)->cursor_needs_physical) {
                         if (intel_crtc->cursor_bo != obj)
                                 i915_gem_detach_phys_object(dev, intel_crtc->cursor_bo);
                 } else
@@ -7638,13 +7850,18 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
 
         mutex_unlock(&dev->struct_mutex);
 
+        old_width = intel_crtc->cursor_width;
+
         intel_crtc->cursor_addr = addr;
         intel_crtc->cursor_bo = obj;
         intel_crtc->cursor_width = width;
         intel_crtc->cursor_height = height;
 
-        if (intel_crtc->active)
+        if (intel_crtc->active) {
+                if (old_width != width)
+                        intel_update_watermarks(crtc);
                 intel_crtc_update_cursor(crtc, intel_crtc->cursor_bo != NULL);
+        }
 
         return 0;
 fail_unpin:
@@ -7690,10 +7907,10 @@ static struct drm_display_mode load_detect_mode = {
                  704, 832, 0, 480, 489, 491, 520, 0, DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC),
 };
 
-static struct drm_framebuffer *
-intel_framebuffer_create(struct drm_device *dev,
-                         struct drm_mode_fb_cmd2 *mode_cmd,
-                         struct drm_i915_gem_object *obj)
+struct drm_framebuffer *
+__intel_framebuffer_create(struct drm_device *dev,
+                           struct drm_mode_fb_cmd2 *mode_cmd,
+                           struct drm_i915_gem_object *obj)
 {
         struct intel_framebuffer *intel_fb;
         int ret;
@@ -7704,12 +7921,7 @@ intel_framebuffer_create(struct drm_device *dev,
                 return ERR_PTR(-ENOMEM);
         }
 
-        ret = i915_mutex_lock_interruptible(dev);
-        if (ret)
-                goto err;
-
         ret = intel_framebuffer_init(dev, intel_fb, mode_cmd, obj);
-        mutex_unlock(&dev->struct_mutex);
         if (ret)
                 goto err;
 
@@ -7721,6 +7933,23 @@ err:
         return ERR_PTR(ret);
 }
 
+static struct drm_framebuffer *
+intel_framebuffer_create(struct drm_device *dev,
+                         struct drm_mode_fb_cmd2 *mode_cmd,
+                         struct drm_i915_gem_object *obj)
+{
+        struct drm_framebuffer *fb;
+        int ret;
+
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                return ERR_PTR(ret);
+        fb = __intel_framebuffer_create(dev, mode_cmd, obj);
+        mutex_unlock(&dev->struct_mutex);
+
+        return fb;
+}
+
 static u32
 intel_framebuffer_pitch_for_width(int width, int bpp)
 {
@@ -7766,14 +7995,16 @@ mode_fits_in_fbdev(struct drm_device *dev,
         struct drm_i915_gem_object *obj;
         struct drm_framebuffer *fb;
 
-        if (dev_priv->fbdev == NULL)
+        if (!dev_priv->fbdev)
                 return NULL;
 
-        obj = dev_priv->fbdev->ifb.obj;
-        if (obj == NULL)
+        if (!dev_priv->fbdev->fb)
                 return NULL;
 
-        fb = &dev_priv->fbdev->ifb.base;
+        obj = dev_priv->fbdev->fb->obj;
+        BUG_ON(!obj);
+
+        fb = &dev_priv->fbdev->fb->base;
         if (fb->pitches[0] < intel_framebuffer_pitch_for_width(mode->hdisplay,
                                                                fb->bits_per_pixel))
                 return NULL;
@@ -7855,6 +8086,8 @@ bool intel_get_load_detect_pipe(struct drm_connector *connector,
         to_intel_connector(connector)->new_encoder = intel_encoder;
 
         intel_crtc = to_intel_crtc(crtc);
+        intel_crtc->new_enabled = true;
+        intel_crtc->new_config = &intel_crtc->config;
         old->dpms_mode = connector->dpms;
         old->load_detect_temp = true;
         old->release_fb = NULL;
@@ -7878,21 +8111,28 @@ bool intel_get_load_detect_pipe(struct drm_connector *connector,
                 DRM_DEBUG_KMS("reusing fbdev for load-detection framebuffer\n");
         if (IS_ERR(fb)) {
                 DRM_DEBUG_KMS("failed to allocate framebuffer for load-detection\n");
-                mutex_unlock(&crtc->mutex);
-                return false;
+                goto fail;
         }
 
         if (intel_set_mode(crtc, mode, 0, 0, fb)) {
                 DRM_DEBUG_KMS("failed to set mode on load-detect pipe\n");
                 if (old->release_fb)
                         old->release_fb->funcs->destroy(old->release_fb);
-                mutex_unlock(&crtc->mutex);
-                return false;
+                goto fail;
         }
 
         /* let the connector get through one full cycle before testing */
         intel_wait_for_vblank(dev, intel_crtc->pipe);
         return true;
+
+ fail:
+        intel_crtc->new_enabled = crtc->enabled;
+        if (intel_crtc->new_enabled)
+                intel_crtc->new_config = &intel_crtc->config;
+        else
+                intel_crtc->new_config = NULL;
+        mutex_unlock(&crtc->mutex);
+        return false;
 }
 
 void intel_release_load_detect_pipe(struct drm_connector *connector,
@@ -7902,6 +8142,7 @@ void intel_release_load_detect_pipe(struct drm_connector *connector,
                 intel_attached_encoder(connector);
         struct drm_encoder *encoder = &intel_encoder->base;
         struct drm_crtc *crtc = encoder->crtc;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
         DRM_DEBUG_KMS("[CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
                       connector->base.id, drm_get_connector_name(connector),
@@ -7910,6 +8151,8 @@ void intel_release_load_detect_pipe(struct drm_connector *connector,
         if (old->load_detect_temp) {
                 to_intel_connector(connector)->new_encoder = NULL;
                 intel_encoder->new_crtc = NULL;
+                intel_crtc->new_enabled = false;
+                intel_crtc->new_config = NULL;
                 intel_set_mode(crtc, NULL, 0, 0, NULL);
 
                 if (old->release_fb) {
@@ -8122,7 +8365,7 @@ struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
 static void intel_increase_pllclock(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         int dpll_reg = DPLL(pipe);
@@ -8153,7 +8396,7 @@ static void intel_increase_pllclock(struct drm_crtc *crtc)
 static void intel_decrease_pllclock(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
         if (HAS_PCH_SPLIT(dev))
@@ -8190,8 +8433,12 @@ void intel_mark_busy(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        hsw_package_c8_gpu_busy(dev_priv);
+        if (dev_priv->mm.busy)
+                return;
+
+        intel_runtime_pm_get(dev_priv);
         i915_update_gfx_val(dev_priv);
+        dev_priv->mm.busy = true;
 }
 
 void intel_mark_idle(struct drm_device *dev)
@@ -8199,20 +8446,26 @@ void intel_mark_idle(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_crtc *crtc;
 
-        hsw_package_c8_gpu_idle(dev_priv);
-
-        if (!i915_powersave)
+        if (!dev_priv->mm.busy)
                 return;
 
+        dev_priv->mm.busy = false;
+
+        if (!i915.powersave)
+                goto out;
+
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-                if (!crtc->fb)
+                if (!crtc->primary->fb)
                         continue;
 
                 intel_decrease_pllclock(crtc);
         }
 
-        if (dev_priv->info->gen >= 6)
+        if (INTEL_INFO(dev)->gen >= 6)
                 gen6_rps_idle(dev->dev_private);
+
+out:
+        intel_runtime_pm_put(dev_priv);
 }
 
 void intel_mark_fb_busy(struct drm_i915_gem_object *obj,
@@ -8221,14 +8474,14 @@ void intel_mark_fb_busy(struct drm_i915_gem_object *obj,
         struct drm_device *dev = obj->base.dev;
         struct drm_crtc *crtc;
 
-        if (!i915_powersave)
+        if (!i915.powersave)
                 return;
 
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-                if (!crtc->fb)
+                if (!crtc->primary->fb)
                         continue;
 
-                if (to_intel_framebuffer(crtc->fb)->obj != obj)
+                if (to_intel_framebuffer(crtc->primary->fb)->obj != obj)
                         continue;
 
                 intel_increase_pllclock(crtc);
@@ -8284,7 +8537,7 @@ static void intel_unpin_work_fn(struct work_struct *__work)
 static void do_intel_finish_page_flip(struct drm_device *dev,
                                       struct drm_crtc *crtc)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_unpin_work *work;
         unsigned long flags;
@@ -8325,7 +8578,7 @@ static void do_intel_finish_page_flip(struct drm_device *dev,
 
 void intel_finish_page_flip(struct drm_device *dev, int pipe)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
 
         do_intel_finish_page_flip(dev, crtc);
@@ -8333,7 +8586,7 @@ void intel_finish_page_flip(struct drm_device *dev, int pipe)
 
 void intel_finish_page_flip_plane(struct drm_device *dev, int plane)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_crtc *crtc = dev_priv->plane_to_crtc_mapping[plane];
 
         do_intel_finish_page_flip(dev, crtc);
@@ -8341,7 +8594,7 @@ void intel_finish_page_flip_plane(struct drm_device *dev, int plane)
 
 void intel_prepare_page_flip(struct drm_device *dev, int plane)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc =
                 to_intel_crtc(dev_priv->plane_to_crtc_mapping[plane]);
         unsigned long flags;
@@ -8656,7 +8909,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_framebuffer *old_fb = crtc->fb;
+        struct drm_framebuffer *old_fb = crtc->primary->fb;
         struct drm_i915_gem_object *obj = to_intel_framebuffer(fb)->obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_unpin_work *work;
@@ -8664,7 +8917,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         int ret;
 
         /* Can't change pixel format via MI display flips. */
-        if (fb->pixel_format != crtc->fb->pixel_format)
+        if (fb->pixel_format != crtc->primary->fb->pixel_format)
                 return -EINVAL;
 
         /*
@@ -8672,10 +8925,13 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
          * Note that pitch changes could also affect these register.
          */
         if (INTEL_INFO(dev)->gen > 3 &&
-            (fb->offsets[0] != crtc->fb->offsets[0] ||
-             fb->pitches[0] != crtc->fb->pitches[0]))
+            (fb->offsets[0] != crtc->primary->fb->offsets[0] ||
+             fb->pitches[0] != crtc->primary->fb->pitches[0]))
                 return -EINVAL;
 
+        if (i915_terminally_wedged(&dev_priv->gpu_error))
+                goto out_hang;
+
         work = kzalloc(sizeof(*work), GFP_KERNEL);
         if (work == NULL)
                 return -ENOMEM;
@@ -8713,7 +8969,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         drm_gem_object_reference(&work->old_fb_obj->base);
         drm_gem_object_reference(&obj->base);
 
-        crtc->fb = fb;
+        crtc->primary->fb = fb;
 
         work->pending_flip_obj = obj;
 
@@ -8736,7 +8992,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 
 cleanup_pending:
         atomic_dec(&intel_crtc->unpin_work_count);
-        crtc->fb = old_fb;
+        crtc->primary->fb = old_fb;
         drm_gem_object_unreference(&work->old_fb_obj->base);
         drm_gem_object_unreference(&obj->base);
         mutex_unlock(&dev->struct_mutex);
@@ -8750,6 +9006,13 @@ cleanup:
 free_work:
         kfree(work);
 
+        if (ret == -EIO) {
+out_hang:
+                intel_crtc_wait_for_pending_flips(crtc);
+                ret = intel_pipe_set_base(crtc, crtc->x, crtc->y, fb);
+                if (ret == 0 && event)
+                        drm_send_vblank_event(dev, intel_crtc->pipe, event);
+        }
         return ret;
 }
 
@@ -8766,6 +9029,7 @@ static struct drm_crtc_helper_funcs intel_helper_funcs = {
  */
 static void intel_modeset_update_staged_output_state(struct drm_device *dev)
 {
+        struct intel_crtc *crtc;
         struct intel_encoder *encoder;
         struct intel_connector *connector;
 
@@ -8780,6 +9044,16 @@ static void intel_modeset_update_staged_output_state(struct drm_device *dev)
                 encoder->new_crtc =
                         to_intel_crtc(encoder->base.crtc);
         }
+
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list,
+                            base.head) {
+                crtc->new_enabled = crtc->base.enabled;
+
+                if (crtc->new_enabled)
+                        crtc->new_config = &crtc->config;
+                else
+                        crtc->new_config = NULL;
+        }
 }
 
 /**
@@ -8789,6 +9063,7 @@ static void intel_modeset_update_staged_output_state(struct drm_device *dev)
  */
 static void intel_modeset_commit_output_state(struct drm_device *dev)
 {
+        struct intel_crtc *crtc;
         struct intel_encoder *encoder;
         struct intel_connector *connector;
 
@@ -8801,6 +9076,11 @@ static void intel_modeset_commit_output_state(struct drm_device *dev)
                             base.head) {
                 encoder->base.crtc = &encoder->new_crtc->base;
         }
+
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list,
+                            base.head) {
+                crtc->base.enabled = crtc->new_enabled;
+        }
 }
 
 static void
@@ -8941,23 +9221,47 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
         DRM_DEBUG_KMS("double wide: %i\n", pipe_config->double_wide);
 }
 
-static bool check_encoder_cloning(struct drm_crtc *crtc)
+static bool encoders_cloneable(const struct intel_encoder *a,
+                               const struct intel_encoder *b)
 {
-        int num_encoders = 0;
-        bool uncloneable_encoders = false;
+        /* masks could be asymmetric, so check both ways */
+        return a == b || (a->cloneable & (1 << b->type) &&
+                          b->cloneable & (1 << a->type));
+}
+
+static bool check_single_encoder_cloning(struct intel_crtc *crtc,
+                                         struct intel_encoder *encoder)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct intel_encoder *source_encoder;
+
+        list_for_each_entry(source_encoder,
+                            &dev->mode_config.encoder_list, base.head) {
+                if (source_encoder->new_crtc != crtc)
+                        continue;
+
+                if (!encoders_cloneable(encoder, source_encoder))
+                        return false;
+        }
+
+        return true;
+}
+
+static bool check_encoder_cloning(struct intel_crtc *crtc)
+{
+        struct drm_device *dev = crtc->base.dev;
         struct intel_encoder *encoder;
 
-        list_for_each_entry(encoder, &crtc->dev->mode_config.encoder_list,
-                            base.head) {
-                if (&encoder->new_crtc->base != crtc)
+        list_for_each_entry(encoder,
+                            &dev->mode_config.encoder_list, base.head) {
+                if (encoder->new_crtc != crtc)
                         continue;
 
-                num_encoders++;
-                if (!encoder->cloneable)
-                        uncloneable_encoders = true;
+                if (!check_single_encoder_cloning(crtc, encoder))
+                        return false;
         }
 
-        return !(num_encoders > 1 && uncloneable_encoders);
+        return true;
 }
 
 static struct intel_crtc_config *
@@ -8971,7 +9275,7 @@ intel_modeset_pipe_config(struct drm_crtc *crtc,
         int plane_bpp, ret = -EINVAL;
         bool retry = true;
 
-        if (!check_encoder_cloning(crtc)) {
+        if (!check_encoder_cloning(to_intel_crtc(crtc))) {
                 DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
                 return ERR_PTR(-EINVAL);
         }
@@ -9127,29 +9431,22 @@ intel_modeset_affected_pipes(struct drm_crtc *crtc, unsigned *modeset_pipes,
                         *prepare_pipes |= 1 << encoder->new_crtc->pipe;
         }
 
-        /* Check for any pipes that will be fully disabled ... */
+        /* Check for pipes that will be enabled/disabled ... */
         list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
                             base.head) {
-                bool used = false;
-
-                /* Don't try to disable disabled crtcs. */
-                if (!intel_crtc->base.enabled)
+                if (intel_crtc->base.enabled == intel_crtc->new_enabled)
                         continue;
 
-                list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-                                    base.head) {
-                        if (encoder->new_crtc == intel_crtc)
-                                used = true;
-                }
-
-                if (!used)
+                if (!intel_crtc->new_enabled)
                         *disable_pipes |= 1 << intel_crtc->pipe;
+                else
+                        *prepare_pipes |= 1 << intel_crtc->pipe;
         }
 
 
         /* set_mode is also used to update properties on life display pipes. */
         intel_crtc = to_intel_crtc(crtc);
-        if (crtc->enabled)
+        if (intel_crtc->new_enabled)
                 *prepare_pipes |= 1 << intel_crtc->pipe;
 
         /*
@@ -9208,10 +9505,13 @@ intel_modeset_update_state(struct drm_device *dev, unsigned prepare_pipes)
 
         intel_modeset_commit_output_state(dev);
 
-        /* Update computed state. */
+        /* Double check state. */
         list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list,
                             base.head) {
-                intel_crtc->base.enabled = intel_crtc_in_use(&intel_crtc->base);
+                WARN_ON(intel_crtc->base.enabled != intel_crtc_in_use(&intel_crtc->base));
+                WARN_ON(intel_crtc->new_config &&
+                        intel_crtc->new_config != &intel_crtc->config);
+                WARN_ON(intel_crtc->base.enabled != !!intel_crtc->new_config);
         }
 
         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
@@ -9380,10 +9680,8 @@ intel_pipe_config_compare(struct drm_device *dev,
         if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5)
                 PIPE_CONF_CHECK_I(pipe_bpp);
 
-        if (!HAS_DDI(dev)) {
-                PIPE_CONF_CHECK_CLOCK_FUZZY(adjusted_mode.crtc_clock);
-                PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
-        }
+        PIPE_CONF_CHECK_CLOCK_FUZZY(adjusted_mode.crtc_clock);
+        PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
 
 #undef PIPE_CONF_CHECK_X
 #undef PIPE_CONF_CHECK_I
@@ -9471,7 +9769,7 @@ check_encoder_state(struct drm_device *dev)
 static void
 check_crtc_state(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *crtc;
         struct intel_encoder *encoder;
         struct intel_crtc_config pipe_config;
@@ -9539,7 +9837,7 @@ check_crtc_state(struct drm_device *dev)
 static void
 check_shared_dpll_state(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *crtc;
         struct intel_dpll_hw_state dpll_hw_state;
         int i;
@@ -9612,7 +9910,7 @@ static int __intel_set_mode(struct drm_crtc *crtc,
                             int x, int y, struct drm_framebuffer *fb)
 {
         struct drm_device *dev = crtc->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_display_mode *saved_mode;
         struct intel_crtc_config *pipe_config = NULL;
         struct intel_crtc *intel_crtc;
@@ -9643,6 +9941,7 @@ static int __intel_set_mode(struct drm_crtc *crtc,
                 }
                 intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config,
                                        "[modeset]");
+                to_intel_crtc(crtc)->new_config = pipe_config;
         }
 
         /*
@@ -9653,8 +9952,7 @@ static int __intel_set_mode(struct drm_crtc *crtc,
          * adjusted_mode bits in the crtc directly.
          */
         if (IS_VALLEYVIEW(dev)) {
-                valleyview_modeset_global_pipes(dev, &prepare_pipes,
-                                                modeset_pipes, pipe_config);
+                valleyview_modeset_global_pipes(dev, &prepare_pipes);
 
                 /* may have added more to prepare_pipes than we should */
                 prepare_pipes &= ~disable_pipes;
@@ -9676,6 +9974,7 @@ static int __intel_set_mode(struct drm_crtc *crtc,
                 /* mode_set/enable/disable functions rely on a correct pipe
                  * config. */
                 to_intel_crtc(crtc)->config = *pipe_config;
+                to_intel_crtc(crtc)->new_config = &to_intel_crtc(crtc)->config;
 
                 /*
                  * Calculate and store various constants which
@@ -9734,7 +10033,7 @@ static int intel_set_mode(struct drm_crtc *crtc,
 
 void intel_crtc_restore_mode(struct drm_crtc *crtc)
 {
-        intel_set_mode(crtc, &crtc->mode, crtc->x, crtc->y, crtc->fb);
+        intel_set_mode(crtc, &crtc->mode, crtc->x, crtc->y, crtc->primary->fb);
 }
 
 #undef for_each_intel_crtc_masked
@@ -9746,16 +10045,24 @@ static void intel_set_config_free(struct intel_set_config *config)
 
         kfree(config->save_connector_encoders);
         kfree(config->save_encoder_crtcs);
+        kfree(config->save_crtc_enabled);
         kfree(config);
 }
 
 static int intel_set_config_save_state(struct drm_device *dev,
                                        struct intel_set_config *config)
 {
+        struct drm_crtc *crtc;
         struct drm_encoder *encoder;
         struct drm_connector *connector;
         int count;
 
+        config->save_crtc_enabled =
+                kcalloc(dev->mode_config.num_crtc,
+                        sizeof(bool), GFP_KERNEL);
+        if (!config->save_crtc_enabled)
+                return -ENOMEM;
+
         config->save_encoder_crtcs =
                 kcalloc(dev->mode_config.num_encoder,
                         sizeof(struct drm_crtc *), GFP_KERNEL);
@@ -9773,6 +10080,11 @@ static int intel_set_config_save_state(struct drm_device *dev,
          * restored, not the drivers personal bookkeeping.
          */
         count = 0;
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                config->save_crtc_enabled[count++] = crtc->enabled;
+        }
+
+        count = 0;
         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
                 config->save_encoder_crtcs[count++] = encoder->crtc;
         }
@@ -9788,11 +10100,22 @@ static int intel_set_config_save_state(struct drm_device *dev,
 static void intel_set_config_restore_state(struct drm_device *dev,
                                            struct intel_set_config *config)
 {
+        struct intel_crtc *crtc;
         struct intel_encoder *encoder;
         struct intel_connector *connector;
         int count;
 
         count = 0;
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, base.head) {
+                crtc->new_enabled = config->save_crtc_enabled[count++];
+
+                if (crtc->new_enabled)
+                        crtc->new_config = &crtc->config;
+                else
+                        crtc->new_config = NULL;
+        }
+
+        count = 0;
         list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
                 encoder->new_crtc =
                         to_intel_crtc(config->save_encoder_crtcs[count++]);
@@ -9834,13 +10157,13 @@ intel_set_config_compute_mode_changes(struct drm_mode_set *set,
          * and then just flip_or_move it */
         if (is_crtc_connector_off(set)) {
                 config->mode_changed = true;
-        } else if (set->crtc->fb != set->fb) {
+        } else if (set->crtc->primary->fb != set->fb) {
                 /* If we have no fb then treat it as a full mode set */
-                if (set->crtc->fb == NULL) {
+                if (set->crtc->primary->fb == NULL) {
                         struct intel_crtc *intel_crtc =
                                 to_intel_crtc(set->crtc);
 
-                        if (intel_crtc->active && i915_fastboot) {
+                        if (intel_crtc->active && i915.fastboot) {
                                 DRM_DEBUG_KMS("crtc has no fb, will flip\n");
                                 config->fb_changed = true;
                         } else {
@@ -9850,7 +10173,7 @@ intel_set_config_compute_mode_changes(struct drm_mode_set *set,
                 } else if (set->fb == NULL) {
                         config->mode_changed = true;
                 } else if (set->fb->pixel_format !=
-                           set->crtc->fb->pixel_format) {
+                           set->crtc->primary->fb->pixel_format) {
                         config->mode_changed = true;
                 } else {
                         config->fb_changed = true;
@@ -9876,9 +10199,9 @@ intel_modeset_stage_output_state(struct drm_device *dev,
                                  struct drm_mode_set *set,
                                  struct intel_set_config *config)
 {
-        struct drm_crtc *new_crtc;
         struct intel_connector *connector;
         struct intel_encoder *encoder;
+        struct intel_crtc *crtc;
         int ro;
 
         /* The upper layers ensure that we either disable a crtc or have a list
@@ -9921,6 +10244,8 @@ intel_modeset_stage_output_state(struct drm_device *dev,
         /* Update crtc of enabled connectors. */
         list_for_each_entry(connector, &dev->mode_config.connector_list,
                             base.head) {
+                struct drm_crtc *new_crtc;
+
                 if (!connector->new_encoder)
                         continue;
 
@@ -9971,9 +10296,58 @@ intel_modeset_stage_output_state(struct drm_device *dev,
         }
         /* Now we've also updated encoder->new_crtc for all encoders. */
 
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list,
+                            base.head) {
+                crtc->new_enabled = false;
+
+                list_for_each_entry(encoder,
+                                    &dev->mode_config.encoder_list,
+                                    base.head) {
+                        if (encoder->new_crtc == crtc) {
+                                crtc->new_enabled = true;
+                                break;
+                        }
+                }
+
+                if (crtc->new_enabled != crtc->base.enabled) {
+                        DRM_DEBUG_KMS("crtc %sabled, full mode switch\n",
+                                      crtc->new_enabled ? "en" : "dis");
+                        config->mode_changed = true;
+                }
+
+                if (crtc->new_enabled)
+                        crtc->new_config = &crtc->config;
+                else
+                        crtc->new_config = NULL;
+        }
+
         return 0;
 }
 
+static void disable_crtc_nofb(struct intel_crtc *crtc)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct intel_encoder *encoder;
+        struct intel_connector *connector;
+
+        DRM_DEBUG_KMS("Trying to restore without FB -> disabling pipe %c\n",
+                      pipe_name(crtc->pipe));
+
+        list_for_each_entry(connector, &dev->mode_config.connector_list, base.head) {
+                if (connector->new_encoder &&
+                    connector->new_encoder->new_crtc == crtc)
+                        connector->new_encoder = NULL;
+        }
+
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
+                if (encoder->new_crtc == crtc)
+                        encoder->new_crtc = NULL;
+        }
+
+        crtc->new_enabled = false;
+        crtc->new_config = NULL;
+}
+
 static int intel_crtc_set_config(struct drm_mode_set *set)
 {
         struct drm_device *dev;
@@ -10012,7 +10386,7 @@ static int intel_crtc_set_config(struct drm_mode_set *set)
         save_set.mode = &set->crtc->mode;
         save_set.x = set->crtc->x;
         save_set.y = set->crtc->y;
-        save_set.fb = set->crtc->fb;
+        save_set.fb = set->crtc->primary->fb;
 
         /* Compute whether we need a full modeset, only an fb base update or no
          * change at all. In the future we might also check whether only the
@@ -10040,7 +10414,7 @@ static int intel_crtc_set_config(struct drm_mode_set *set)
                  * flipping, so increasing its cost here shouldn't be a big
                  * deal).
                  */
-                if (i915_fastboot && ret == 0)
+                if (i915.fastboot && ret == 0)
                         intel_modeset_check_state(set->crtc->dev);
         }
 
@@ -10050,6 +10424,15 @@ static int intel_crtc_set_config(struct drm_mode_set *set)
 fail:
                 intel_set_config_restore_state(dev, config);
 
+                /*
+                 * HACK: if the pipe was on, but we didn't have a framebuffer,
+                 * force the pipe off to avoid oopsing in the modeset code
+                 * due to fb==NULL. This should only happen during boot since
+                 * we don't yet reconstruct the FB from the hardware state.
+                 */
+                if (to_intel_crtc(save_set.crtc)->new_enabled && !save_set.fb)
+                        disable_crtc_nofb(to_intel_crtc(save_set.crtc));
+
                 /* Try to restore the config */
                 if (config->mode_changed &&
                     intel_set_mode(save_set.crtc, save_set.mode,
@@ -10174,7 +10557,7 @@ static void intel_shared_dpll_init(struct drm_device *dev)
 
 static void intel_crtc_init(struct drm_device *dev, int pipe)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc;
         int i;
 
@@ -10184,6 +10567,16 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
 
         drm_crtc_init(dev, &intel_crtc->base, &intel_crtc_funcs);
 
+        if (IS_GEN2(dev)) {
+                intel_crtc->max_cursor_width = GEN2_CURSOR_WIDTH;
+                intel_crtc->max_cursor_height = GEN2_CURSOR_HEIGHT;
+        } else {
+                intel_crtc->max_cursor_width = CURSOR_WIDTH;
+                intel_crtc->max_cursor_height = CURSOR_HEIGHT;
+        }
+        dev->mode_config.cursor_width = intel_crtc->max_cursor_width;
+        dev->mode_config.cursor_height = intel_crtc->max_cursor_height;
+
         drm_mode_crtc_set_gamma_size(&intel_crtc->base, 256);
         for (i = 0; i < 256; i++) {
                 intel_crtc->lut_r[i] = i;
@@ -10255,12 +10648,7 @@ static int intel_encoder_clones(struct intel_encoder *encoder)
 
         list_for_each_entry(source_encoder,
                             &dev->mode_config.encoder_list, base.head) {
-
-                if (encoder == source_encoder)
-                        index_mask |= (1 << entry);
-
-                /* Intel hw has only one MUX where enocoders could be cloned. */
-                if (encoder->cloneable && source_encoder->cloneable)
+                if (encoders_cloneable(encoder, source_encoder))
                         index_mask |= (1 << entry);
 
                 entry++;
@@ -10279,8 +10667,7 @@ static bool has_edp_a(struct drm_device *dev)
         if ((I915_READ(DP_A) & DP_DETECTED) == 0)
                 return false;
 
-        if (IS_GEN5(dev) &&
-            (I915_READ(ILK_DISPLAY_CHICKEN_FUSES) & ILK_eDP_A_DISABLE))
+        if (IS_GEN5(dev) && (I915_READ(FUSE_STRAP) & ILK_eDP_A_DISABLE))
                 return false;
 
         return true;
@@ -10433,18 +10820,13 @@ static void intel_setup_outputs(struct drm_device *dev)
         drm_helper_move_panel_connectors_to_head(dev);
 }
 
-void intel_framebuffer_fini(struct intel_framebuffer *fb)
-{
-        drm_framebuffer_cleanup(&fb->base);
-        WARN_ON(!fb->obj->framebuffer_references--);
-        drm_gem_object_unreference_unlocked(&fb->obj->base);
-}
-
 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
 
-        intel_framebuffer_fini(intel_fb);
+        drm_framebuffer_cleanup(fb);
+        WARN_ON(!intel_fb->obj->framebuffer_references--);
+        drm_gem_object_unreference_unlocked(&intel_fb->obj->base);
         kfree(intel_fb);
 }
 
@@ -10463,12 +10845,12 @@ static const struct drm_framebuffer_funcs intel_fb_funcs = {
         .create_handle = intel_user_framebuffer_create_handle,
 };
 
-int intel_framebuffer_init(struct drm_device *dev,
-                           struct intel_framebuffer *intel_fb,
-                           struct drm_mode_fb_cmd2 *mode_cmd,
-                           struct drm_i915_gem_object *obj)
+static int intel_framebuffer_init(struct drm_device *dev,
+                                  struct intel_framebuffer *intel_fb,
+                                  struct drm_mode_fb_cmd2 *mode_cmd,
+                                  struct drm_i915_gem_object *obj)
 {
-        int aligned_height, tile_height;
+        int aligned_height;
         int pitch_limit;
         int ret;
 
@@ -10562,9 +10944,8 @@ int intel_framebuffer_init(struct drm_device *dev,
         if (mode_cmd->offsets[0] != 0)
                 return -EINVAL;
 
-        tile_height = IS_GEN2(dev) ? 16 : 8;
-        aligned_height = ALIGN(mode_cmd->height,
-                               obj->tiling_mode ? tile_height : 1);
+        aligned_height = intel_align_height(dev, mode_cmd->height,
+                                            obj->tiling_mode);
         /* FIXME drm helper for size checks (especially planar formats)? */
         if (obj->base.size < aligned_height * mode_cmd->pitches[0])
                 return -EINVAL;
@@ -10624,32 +11005,40 @@ static void intel_init_display(struct drm_device *dev)
 
         if (HAS_DDI(dev)) {
                 dev_priv->display.get_pipe_config = haswell_get_pipe_config;
+                dev_priv->display.get_plane_config = ironlake_get_plane_config;
                 dev_priv->display.crtc_mode_set = haswell_crtc_mode_set;
                 dev_priv->display.crtc_enable = haswell_crtc_enable;
                 dev_priv->display.crtc_disable = haswell_crtc_disable;
                 dev_priv->display.off = haswell_crtc_off;
-                dev_priv->display.update_plane = ironlake_update_plane;
+                dev_priv->display.update_primary_plane =
+                        ironlake_update_primary_plane;
         } else if (HAS_PCH_SPLIT(dev)) {
                 dev_priv->display.get_pipe_config = ironlake_get_pipe_config;
+                dev_priv->display.get_plane_config = ironlake_get_plane_config;
                 dev_priv->display.crtc_mode_set = ironlake_crtc_mode_set;
                 dev_priv->display.crtc_enable = ironlake_crtc_enable;
                 dev_priv->display.crtc_disable = ironlake_crtc_disable;
                 dev_priv->display.off = ironlake_crtc_off;
-                dev_priv->display.update_plane = ironlake_update_plane;
+                dev_priv->display.update_primary_plane =
+                        ironlake_update_primary_plane;
         } else if (IS_VALLEYVIEW(dev)) {
                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
+                dev_priv->display.get_plane_config = i9xx_get_plane_config;
                 dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set;
                 dev_priv->display.crtc_enable = valleyview_crtc_enable;
                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
                 dev_priv->display.off = i9xx_crtc_off;
-                dev_priv->display.update_plane = i9xx_update_plane;
+                dev_priv->display.update_primary_plane =
+                        i9xx_update_primary_plane;
         } else {
                 dev_priv->display.get_pipe_config = i9xx_get_pipe_config;
+                dev_priv->display.get_plane_config = i9xx_get_plane_config;
                 dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set;
                 dev_priv->display.crtc_enable = i9xx_crtc_enable;
                 dev_priv->display.crtc_disable = i9xx_crtc_disable;
                 dev_priv->display.off = i9xx_crtc_off;
-                dev_priv->display.update_plane = i9xx_update_plane;
+                dev_priv->display.update_primary_plane =
+                        i9xx_update_primary_plane;
         }
 
         /* Returns the core display clock speed */
@@ -10839,6 +11228,9 @@ static struct intel_quirk intel_quirks[] = {
 
         /* Acer Aspire 4736Z */
         { 0x2a42, 0x1025, 0x0260, quirk_invert_brightness },
+
+        /* Acer Aspire 5336 */
+        { 0x2a42, 0x1025, 0x048a, quirk_invert_brightness },
 };
 
 static void intel_init_quirks(struct drm_device *dev)
@@ -10869,6 +11261,7 @@ static void i915_disable_vga(struct drm_device *dev)
         u8 sr1;
         u32 vga_reg = i915_vgacntrl_reg(dev);
 
+        /* WaEnableVGAAccessThroughIOPort:ctg,elk,ilk,snb,ivb,vlv,hsw */
         vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
         outb(SR01, VGA_SR_INDEX);
         sr1 = inb(VGA_SR_DATA);
@@ -10901,7 +11294,9 @@ void intel_modeset_suspend_hw(struct drm_device *dev)
 void intel_modeset_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int i, j, ret;
+        int sprite, ret;
+        enum pipe pipe;
+        struct intel_crtc *crtc;
 
         drm_mode_config_init(dev);
 
@@ -10938,13 +11333,13 @@ void intel_modeset_init(struct drm_device *dev)
                       INTEL_INFO(dev)->num_pipes,
                       INTEL_INFO(dev)->num_pipes > 1 ? "s" : "");
 
-        for_each_pipe(i) {
-                intel_crtc_init(dev, i);
-                for (j = 0; j < dev_priv->num_plane; j++) {
-                        ret = intel_plane_init(dev, i, j);
+        for_each_pipe(pipe) {
+                intel_crtc_init(dev, pipe);
+                for_each_sprite(pipe, sprite) {
+                        ret = intel_plane_init(dev, pipe, sprite);
                         if (ret)
                                 DRM_DEBUG_KMS("pipe %c sprite %c init failed: %d\n",
-                                              pipe_name(i), sprite_name(i, j), ret);
+                                              pipe_name(pipe), sprite_name(pipe, sprite), ret);
                 }
         }
 
@@ -10960,6 +11355,33 @@ void intel_modeset_init(struct drm_device *dev)
 
         /* Just in case the BIOS is doing something questionable. */
         intel_disable_fbc(dev);
+
+        mutex_lock(&dev->mode_config.mutex);
+        intel_modeset_setup_hw_state(dev, false);
+        mutex_unlock(&dev->mode_config.mutex);
+
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list,
+                            base.head) {
+                if (!crtc->active)
+                        continue;
+
+                /*
+                 * Note that reserving the BIOS fb up front prevents us
+                 * from stuffing other stolen allocations like the ring
+                 * on top.  This prevents some ugliness at boot time, and
+                 * can even allow for smooth boot transitions if the BIOS
+                 * fb is large enough for the active pipe configuration.
+                 */
+                if (dev_priv->display.get_plane_config) {
+                        dev_priv->display.get_plane_config(crtc,
+                                                           &crtc->plane_config);
+                        /*
+                         * If the fb is shared between multiple heads, we'll
+                         * just get the first one.
+                         */
+                        intel_find_plane_obj(crtc, &crtc->plane_config);
+                }
+        }
 }
 
 static void
@@ -11097,6 +11519,17 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
                         encoder->base.crtc = NULL;
                 }
         }
+        if (crtc->active) {
+                /*
+                 * We start out with underrun reporting disabled to avoid races.
+                 * For correct bookkeeping mark this on active crtcs.
+                 *
+                 * No protection against concurrent access is required - at
+                 * worst a fifo underrun happens which also sets this to false.
+                 */
+                crtc->cpu_fifo_underrun_disabled = true;
+                crtc->pch_fifo_underrun_disabled = true;
+        }
 }
 
 static void intel_sanitize_encoder(struct intel_encoder *encoder)
@@ -11142,11 +11575,21 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder)
          * the crtc fixup. */
 }
 
-void i915_redisable_vga(struct drm_device *dev)
+void i915_redisable_vga_power_on(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 vga_reg = i915_vgacntrl_reg(dev);
 
+        if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
+                DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
+                i915_disable_vga(dev);
+        }
+}
+
+void i915_redisable_vga(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
         /* This function can be called both from intel_modeset_setup_hw_state or
          * at a very early point in our resume sequence, where the power well
          * structures are not yet restored. Since this function is at a very
@@ -11154,14 +11597,10 @@ void i915_redisable_vga(struct drm_device *dev)
          * level, just check if the power well is enabled instead of trying to
          * follow the "don't touch the power well if we don't need it" policy
          * the rest of the driver uses. */
-        if ((IS_HASWELL(dev) || IS_BROADWELL(dev)) &&
-            (I915_READ(HSW_PWR_WELL_DRIVER) & HSW_PWR_WELL_STATE_ENABLED) == 0)
+        if (!intel_display_power_enabled(dev_priv, POWER_DOMAIN_VGA))
                 return;
 
-        if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
-                DRM_DEBUG_KMS("Something enabled VGA plane, disabling it\n");
-                i915_disable_vga(dev);
-        }
+        i915_redisable_vga_power_on(dev);
 }
 
 static void intel_modeset_readout_hw_state(struct drm_device *dev)
@@ -11265,9 +11704,8 @@ void intel_modeset_setup_hw_state(struct drm_device *dev,
          */
         list_for_each_entry(crtc, &dev->mode_config.crtc_list,
                             base.head) {
-                if (crtc->active && i915_fastboot) {
-                        intel_crtc_mode_from_pipe_config(crtc, &crtc->config);
-
+                if (crtc->active && i915.fastboot) {
+                        intel_mode_from_pipe_config(&crtc->base.mode, &crtc->config);
                         DRM_DEBUG_KMS("[CRTC:%d] found active mode: ",
                                       crtc->base.base.id);
                         drm_mode_debug_printmodeline(&crtc->base.mode);
@@ -11313,7 +11751,7 @@ void intel_modeset_setup_hw_state(struct drm_device *dev,
                                 dev_priv->pipe_to_crtc_mapping[pipe];
 
                         __intel_set_mode(crtc, &crtc->mode, crtc->x, crtc->y,
-                                         crtc->fb);
+                                         crtc->primary->fb);
                 }
         } else {
                 intel_modeset_update_staged_output_state(dev);
@@ -11324,14 +11762,44 @@ void intel_modeset_setup_hw_state(struct drm_device *dev,
 
 void intel_modeset_gem_init(struct drm_device *dev)
 {
+        struct drm_crtc *c;
+        struct intel_framebuffer *fb;
+
+        mutex_lock(&dev->struct_mutex);
+        intel_init_gt_powersave(dev);
+        mutex_unlock(&dev->struct_mutex);
+
         intel_modeset_init_hw(dev);
 
         intel_setup_overlay(dev);
 
-        mutex_lock(&dev->mode_config.mutex);
-        drm_mode_config_reset(dev);
-        intel_modeset_setup_hw_state(dev, false);
-        mutex_unlock(&dev->mode_config.mutex);
+        /*
+         * Make sure any fbs we allocated at startup are properly
+         * pinned & fenced.  When we do the allocation it's too early
+         * for this.
+         */
+        mutex_lock(&dev->struct_mutex);
+        list_for_each_entry(c, &dev->mode_config.crtc_list, head) {
+                if (!c->primary->fb)
+                        continue;
+
+                fb = to_intel_framebuffer(c->primary->fb);
+                if (intel_pin_and_fence_fb_obj(dev, fb->obj, NULL)) {
+                        DRM_ERROR("failed to pin boot fb on pipe %d\n",
+                                  to_intel_crtc(c)->pipe);
+                        drm_framebuffer_unreference(c->primary->fb);
+                        c->primary->fb = NULL;
+                }
+        }
+        mutex_unlock(&dev->struct_mutex);
+}
+
+void intel_connector_unregister(struct intel_connector *intel_connector)
+{
+        struct drm_connector *connector = &intel_connector->base;
+
+        intel_panel_destroy_backlight(connector);
+        drm_sysfs_connector_remove(connector);
 }
 
 void intel_modeset_cleanup(struct drm_device *dev)
@@ -11359,7 +11827,7 @@ void intel_modeset_cleanup(struct drm_device *dev)
 
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                 /* Skip inactive CRTCs */
-                if (!crtc->fb)
+                if (!crtc->primary->fb)
                         continue;
 
                 intel_increase_pllclock(crtc);
@@ -11378,13 +11846,19 @@ void intel_modeset_cleanup(struct drm_device *dev)
 
         /* destroy the backlight and sysfs files before encoders/connectors */
         list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                intel_panel_destroy_backlight(connector);
-                drm_sysfs_connector_remove(connector);
+                struct intel_connector *intel_connector;
+
+                intel_connector = to_intel_connector(connector);
+                intel_connector->unregister(intel_connector);
         }
 
         drm_mode_config_cleanup(dev);
 
         intel_cleanup_overlay(dev);
+
+        mutex_lock(&dev->struct_mutex);
+        intel_cleanup_gt_powersave(dev);
+        mutex_unlock(&dev->struct_mutex);
 }
 
 /*
@@ -11412,12 +11886,24 @@ int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
         unsigned reg = INTEL_INFO(dev)->gen >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
         u16 gmch_ctrl;
 
-        pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl);
+        if (pci_read_config_word(dev_priv->bridge_dev, reg, &gmch_ctrl)) {
+                DRM_ERROR("failed to read control word\n");
+                return -EIO;
+        }
+
+        if (!!(gmch_ctrl & INTEL_GMCH_VGA_DISABLE) == !state)
+                return 0;
+
         if (state)
                 gmch_ctrl &= ~INTEL_GMCH_VGA_DISABLE;
         else
                 gmch_ctrl |= INTEL_GMCH_VGA_DISABLE;
-        pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl);
+
+        if (pci_write_config_word(dev_priv->bridge_dev, reg, gmch_ctrl)) {
+                DRM_ERROR("failed to write control word\n");
+                return -EIO;
+        }
+
         return 0;
 }
 
@@ -11467,7 +11953,7 @@ struct intel_display_error_state {
 struct intel_display_error_state *
 intel_display_capture_error_state(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_display_error_state *error;
         int transcoders[] = {
                 TRANSCODER_A,
@@ -11489,7 +11975,8 @@ intel_display_capture_error_state(struct drm_device *dev)
 
         for_each_pipe(i) {
                 error->pipe[i].power_domain_on =
-                        intel_display_power_enabled_sw(dev, POWER_DOMAIN_PIPE(i));
+                        intel_display_power_enabled_sw(dev_priv,
+                                                       POWER_DOMAIN_PIPE(i));
                 if (!error->pipe[i].power_domain_on)
                         continue;
 
@@ -11527,7 +12014,7 @@ intel_display_capture_error_state(struct drm_device *dev)
                 enum transcoder cpu_transcoder = transcoders[i];
 
                 error->transcoder[i].power_domain_on =
-                        intel_display_power_enabled_sw(dev,
+                        intel_display_power_enabled_sw(dev_priv,
                                 POWER_DOMAIN_TRANSCODER(cpu_transcoder));
                 if (!error->transcoder[i].power_domain_on)
                         continue;
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 2688f6d64bb9..a0dad1a2f819 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -91,18 +91,25 @@ static struct intel_dp *intel_attached_dp(struct drm_connector *connector)
 }
 
 static void intel_dp_link_down(struct intel_dp *intel_dp);
+static bool _edp_panel_vdd_on(struct intel_dp *intel_dp);
+static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync);
 
 static int
 intel_dp_max_link_bw(struct intel_dp *intel_dp)
 {
         int max_link_bw = intel_dp->dpcd[DP_MAX_LINK_RATE];
+        struct drm_device *dev = intel_dp->attached_connector->base.dev;
 
         switch (max_link_bw) {
         case DP_LINK_BW_1_62:
         case DP_LINK_BW_2_7:
                 break;
         case DP_LINK_BW_5_4: /* 1.2 capable displays may advertise higher bw */
-                max_link_bw = DP_LINK_BW_2_7;
+                if ((IS_HASWELL(dev) || INTEL_INFO(dev)->gen >= 8) &&
+                    intel_dp->dpcd[DP_DPCD_REV] >= 0x12)
+                        max_link_bw = DP_LINK_BW_5_4;
+                else
+                        max_link_bw = DP_LINK_BW_2_7;
                 break;
         default:
                 WARN(1, "invalid max DP link bw val %x, using 1.62Gbps\n",
@@ -294,7 +301,7 @@ static u32 _pp_stat_reg(struct intel_dp *intel_dp)
                 return VLV_PIPE_PP_STATUS(vlv_power_sequencer_pipe(intel_dp));
 }
 
-static bool ironlake_edp_have_panel_power(struct intel_dp *intel_dp)
+static bool edp_have_panel_power(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -302,12 +309,13 @@ static bool ironlake_edp_have_panel_power(struct intel_dp *intel_dp)
         return (I915_READ(_pp_stat_reg(intel_dp)) & PP_ON) != 0;
 }
 
-static bool ironlake_edp_have_panel_vdd(struct intel_dp *intel_dp)
+static bool edp_have_panel_vdd(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        return (I915_READ(_pp_ctrl_reg(intel_dp)) & EDP_FORCE_VDD) != 0;
+        return !dev_priv->pm.suspended &&
+               (I915_READ(_pp_ctrl_reg(intel_dp)) & EDP_FORCE_VDD) != 0;
 }
 
 static void
@@ -319,7 +327,7 @@ intel_dp_check_edp(struct intel_dp *intel_dp)
         if (!is_edp(intel_dp))
                 return;
 
-        if (!ironlake_edp_have_panel_power(intel_dp) && !ironlake_edp_have_panel_vdd(intel_dp)) {
+        if (!edp_have_panel_power(intel_dp) && !edp_have_panel_vdd(intel_dp)) {
                 WARN(1, "eDP powered off while attempting aux channel communication.\n");
                 DRM_DEBUG_KMS("Status 0x%08x Control 0x%08x\n",
                               I915_READ(_pp_stat_reg(intel_dp)),
@@ -351,31 +359,46 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq)
         return status;
 }
 
-static uint32_t get_aux_clock_divider(struct intel_dp *intel_dp,
-                                      int index)
+static uint32_t i9xx_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
 
-        /* The clock divider is based off the hrawclk,
-         * and would like to run at 2MHz. So, take the
-         * hrawclk value and divide by 2 and use that
-         *
-         * Note that PCH attached eDP panels should use a 125MHz input
-         * clock divider.
+        /*
+         * The clock divider is based off the hrawclk, and would like to run at
+         * 2MHz.  So, take the hrawclk value and divide by 2 and use that
          */
-        if (IS_VALLEYVIEW(dev)) {
-                return index ? 0 : 100;
-        } else if (intel_dig_port->port == PORT_A) {
-                if (index)
-                        return 0;
-                if (HAS_DDI(dev))
-                        return DIV_ROUND_CLOSEST(intel_ddi_get_cdclk_freq(dev_priv), 2000);
-                else if (IS_GEN6(dev) || IS_GEN7(dev))
+        return index ? 0 : intel_hrawclk(dev) / 2;
+}
+
+static uint32_t ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
+{
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct drm_device *dev = intel_dig_port->base.base.dev;
+
+        if (index)
+                return 0;
+
+        if (intel_dig_port->port == PORT_A) {
+                if (IS_GEN6(dev) || IS_GEN7(dev))
                         return 200; /* SNB & IVB eDP input clock at 400Mhz */
                 else
                         return 225; /* eDP input clock at 450Mhz */
+        } else {
+                return DIV_ROUND_UP(intel_pch_rawclk(dev), 2);
+        }
+}
+
+static uint32_t hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
+{
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct drm_device *dev = intel_dig_port->base.base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (intel_dig_port->port == PORT_A) {
+                if (index)
+                        return 0;
+                return DIV_ROUND_CLOSEST(intel_ddi_get_cdclk_freq(dev_priv), 2000);
         } else if (dev_priv->pch_id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
                 /* Workaround for non-ULT HSW */
                 switch (index) {
@@ -383,13 +406,46 @@ static uint32_t get_aux_clock_divider(struct intel_dp *intel_dp,
                 case 1: return 72;
                 default: return 0;
                 }
-        } else if (HAS_PCH_SPLIT(dev)) {
+        } else  {
                 return index ? 0 : DIV_ROUND_UP(intel_pch_rawclk(dev), 2);
-        } else {
-                return index ? 0 :intel_hrawclk(dev) / 2;
         }
 }
 
+static uint32_t vlv_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
+{
+        return index ? 0 : 100;
+}
+
+static uint32_t i9xx_get_aux_send_ctl(struct intel_dp *intel_dp,
+                                      bool has_aux_irq,
+                                      int send_bytes,
+                                      uint32_t aux_clock_divider)
+{
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct drm_device *dev = intel_dig_port->base.base.dev;
+        uint32_t precharge, timeout;
+
+        if (IS_GEN6(dev))
+                precharge = 3;
+        else
+                precharge = 5;
+
+        if (IS_BROADWELL(dev) && intel_dp->aux_ch_ctl_reg == DPA_AUX_CH_CTL)
+                timeout = DP_AUX_CH_CTL_TIME_OUT_600us;
+        else
+                timeout = DP_AUX_CH_CTL_TIME_OUT_400us;
+
+        return DP_AUX_CH_CTL_SEND_BUSY |
+               DP_AUX_CH_CTL_DONE |
+               (has_aux_irq ? DP_AUX_CH_CTL_INTERRUPT : 0) |
+               DP_AUX_CH_CTL_TIME_OUT_ERROR |
+               timeout |
+               DP_AUX_CH_CTL_RECEIVE_ERROR |
+               (send_bytes << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) |
+               (precharge << DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT) |
+               (aux_clock_divider << DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT);
+}
+
 static int
 intel_dp_aux_ch(struct intel_dp *intel_dp,
                 uint8_t *send, int send_bytes,
@@ -403,9 +459,11 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
         uint32_t aux_clock_divider;
         int i, ret, recv_bytes;
         uint32_t status;
-        int try, precharge, clock = 0;
+        int try, clock = 0;
         bool has_aux_irq = HAS_AUX_IRQ(dev);
-        uint32_t timeout;
+        bool vdd;
+
+        vdd = _edp_panel_vdd_on(intel_dp);
 
         /* dp aux is extremely sensitive to irq latency, hence request the
          * lowest possible wakeup latency and so prevent the cpu from going into
@@ -415,16 +473,6 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
 
         intel_dp_check_edp(intel_dp);
 
-        if (IS_GEN6(dev))
-                precharge = 3;
-        else
-                precharge = 5;
-
-        if (IS_BROADWELL(dev) && ch_ctl == DPA_AUX_CH_CTL)
-                timeout = DP_AUX_CH_CTL_TIME_OUT_600us;
-        else
-                timeout = DP_AUX_CH_CTL_TIME_OUT_400us;
-
         intel_aux_display_runtime_get(dev_priv);
 
         /* Try to wait for any previous AUX channel activity */
@@ -448,7 +496,12 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
                 goto out;
         }
 
-        while ((aux_clock_divider = get_aux_clock_divider(intel_dp, clock++))) {
+        while ((aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, clock++))) {
+                u32 send_ctl = intel_dp->get_aux_send_ctl(intel_dp,
+                                                          has_aux_irq,
+                                                          send_bytes,
+                                                          aux_clock_divider);
+
                 /* Must try at least 3 times according to DP spec */
                 for (try = 0; try < 5; try++) {
                         /* Load the send data into the aux channel data registers */
@@ -457,16 +510,7 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
                                            pack_aux(send + i, send_bytes - i));
 
                         /* Send the command and wait for it to complete */
-                        I915_WRITE(ch_ctl,
-                                   DP_AUX_CH_CTL_SEND_BUSY |
-                                   (has_aux_irq ? DP_AUX_CH_CTL_INTERRUPT : 0) |
-                                   timeout |
-                                   (send_bytes << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) |
-                                   (precharge << DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT) |
-                                   (aux_clock_divider << DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT) |
-                                   DP_AUX_CH_CTL_DONE |
-                                   DP_AUX_CH_CTL_TIME_OUT_ERROR |
-                                   DP_AUX_CH_CTL_RECEIVE_ERROR);
+                        I915_WRITE(ch_ctl, send_ctl);
 
                         status = intel_dp_aux_wait_done(intel_dp, has_aux_irq);
 
@@ -525,246 +569,140 @@ out:
         pm_qos_update_request(&dev_priv->pm_qos, PM_QOS_DEFAULT_VALUE);
         intel_aux_display_runtime_put(dev_priv);
 
+        if (vdd)
+                edp_panel_vdd_off(intel_dp, false);
+
         return ret;
 }
 
-/* Write data to the aux channel in native mode */
-static int
-intel_dp_aux_native_write(struct intel_dp *intel_dp,
-                          uint16_t address, uint8_t *send, int send_bytes)
+#define HEADER_SIZE     4
+static ssize_t
+intel_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
 {
+        struct intel_dp *intel_dp = container_of(aux, struct intel_dp, aux);
+        uint8_t txbuf[20], rxbuf[20];
+        size_t txsize, rxsize;
         int ret;
-        uint8_t msg[20];
-        int msg_bytes;
-        uint8_t ack;
-        int retry;
 
-        if (WARN_ON(send_bytes > 16))
-                return -E2BIG;
+        txbuf[0] = msg->request << 4;
+        txbuf[1] = msg->address >> 8;
+        txbuf[2] = msg->address & 0xff;
+        txbuf[3] = msg->size - 1;
 
-        intel_dp_check_edp(intel_dp);
-        msg[0] = DP_AUX_NATIVE_WRITE << 4;
-        msg[1] = address >> 8;
-        msg[2] = address & 0xff;
-        msg[3] = send_bytes - 1;
-        memcpy(&msg[4], send, send_bytes);
-        msg_bytes = send_bytes + 4;
-        for (retry = 0; retry < 7; retry++) {
-                ret = intel_dp_aux_ch(intel_dp, msg, msg_bytes, &ack, 1);
-                if (ret < 0)
-                        return ret;
-                ack >>= 4;
-                if ((ack & DP_AUX_NATIVE_REPLY_MASK) == DP_AUX_NATIVE_REPLY_ACK)
-                        return send_bytes;
-                else if ((ack & DP_AUX_NATIVE_REPLY_MASK) == DP_AUX_NATIVE_REPLY_DEFER)
-                        usleep_range(400, 500);
-                else
-                        return -EIO;
-        }
+        switch (msg->request & ~DP_AUX_I2C_MOT) {
+        case DP_AUX_NATIVE_WRITE:
+        case DP_AUX_I2C_WRITE:
+                txsize = HEADER_SIZE + msg->size;
+                rxsize = 1;
 
-        DRM_ERROR("too many retries, giving up\n");
-        return -EIO;
-}
+                if (WARN_ON(txsize > 20))
+                        return -E2BIG;
 
-/* Write a single byte to the aux channel in native mode */
-static int
-intel_dp_aux_native_write_1(struct intel_dp *intel_dp,
-                            uint16_t address, uint8_t byte)
-{
-        return intel_dp_aux_native_write(intel_dp, address, &byte, 1);
-}
+                memcpy(txbuf + HEADER_SIZE, msg->buffer, msg->size);
 
-/* read bytes from a native aux channel */
-static int
-intel_dp_aux_native_read(struct intel_dp *intel_dp,
-                         uint16_t address, uint8_t *recv, int recv_bytes)
-{
-        uint8_t msg[4];
-        int msg_bytes;
-        uint8_t reply[20];
-        int reply_bytes;
-        uint8_t ack;
-        int ret;
-        int retry;
+                ret = intel_dp_aux_ch(intel_dp, txbuf, txsize, rxbuf, rxsize);
+                if (ret > 0) {
+                        msg->reply = rxbuf[0] >> 4;
+
+                        /* Return payload size. */
+                        ret = msg->size;
+                }
+                break;
 
-        if (WARN_ON(recv_bytes > 19))
-                return -E2BIG;
+        case DP_AUX_NATIVE_READ:
+        case DP_AUX_I2C_READ:
+                txsize = HEADER_SIZE;
+                rxsize = msg->size + 1;
 
-        intel_dp_check_edp(intel_dp);
-        msg[0] = DP_AUX_NATIVE_READ << 4;
-        msg[1] = address >> 8;
-        msg[2] = address & 0xff;
-        msg[3] = recv_bytes - 1;
-
-        msg_bytes = 4;
-        reply_bytes = recv_bytes + 1;
-
-        for (retry = 0; retry < 7; retry++) {
-                ret = intel_dp_aux_ch(intel_dp, msg, msg_bytes,
-                                      reply, reply_bytes);
-                if (ret == 0)
-                        return -EPROTO;
-                if (ret < 0)
-                        return ret;
-                ack = reply[0] >> 4;
-                if ((ack & DP_AUX_NATIVE_REPLY_MASK) == DP_AUX_NATIVE_REPLY_ACK) {
-                        memcpy(recv, reply + 1, ret - 1);
-                        return ret - 1;
+                if (WARN_ON(rxsize > 20))
+                        return -E2BIG;
+
+                ret = intel_dp_aux_ch(intel_dp, txbuf, txsize, rxbuf, rxsize);
+                if (ret > 0) {
+                        msg->reply = rxbuf[0] >> 4;
+                        /*
+                         * Assume happy day, and copy the data. The caller is
+                         * expected to check msg->reply before touching it.
+                         *
+                         * Return payload size.
+                         */
+                        ret--;
+                        memcpy(msg->buffer, rxbuf + 1, ret);
                 }
-                else if ((ack & DP_AUX_NATIVE_REPLY_MASK) == DP_AUX_NATIVE_REPLY_DEFER)
-                        usleep_range(400, 500);
-                else
-                        return -EIO;
+                break;
+
+        default:
+                ret = -EINVAL;
+                break;
         }
 
-        DRM_ERROR("too many retries, giving up\n");
-        return -EIO;
+        return ret;
 }
 
-static int
-intel_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
-                    uint8_t write_byte, uint8_t *read_byte)
-{
-        struct i2c_algo_dp_aux_data *algo_data = adapter->algo_data;
-        struct intel_dp *intel_dp = container_of(adapter,
-                                                struct intel_dp,
-                                                adapter);
-        uint16_t address = algo_data->address;
-        uint8_t msg[5];
-        uint8_t reply[2];
-        unsigned retry;
-        int msg_bytes;
-        int reply_bytes;
+static void
+intel_dp_aux_init(struct intel_dp *intel_dp, struct intel_connector *connector)
+{
+        struct drm_device *dev = intel_dp_to_dev(intel_dp);
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        enum port port = intel_dig_port->port;
+        const char *name = NULL;
         int ret;
 
-        ironlake_edp_panel_vdd_on(intel_dp);
-        intel_dp_check_edp(intel_dp);
-        /* Set up the command byte */
-        if (mode & MODE_I2C_READ)
-                msg[0] = DP_AUX_I2C_READ << 4;
-        else
-                msg[0] = DP_AUX_I2C_WRITE << 4;
-
-        if (!(mode & MODE_I2C_STOP))
-                msg[0] |= DP_AUX_I2C_MOT << 4;
-
-        msg[1] = address >> 8;
-        msg[2] = address;
-
-        switch (mode) {
-        case MODE_I2C_WRITE:
-                msg[3] = 0;
-                msg[4] = write_byte;
-                msg_bytes = 5;
-                reply_bytes = 1;
+        switch (port) {
+        case PORT_A:
+                intel_dp->aux_ch_ctl_reg = DPA_AUX_CH_CTL;
+                name = "DPDDC-A";
+                break;
+        case PORT_B:
+                intel_dp->aux_ch_ctl_reg = PCH_DPB_AUX_CH_CTL;
+                name = "DPDDC-B";
+                break;
+        case PORT_C:
+                intel_dp->aux_ch_ctl_reg = PCH_DPC_AUX_CH_CTL;
+                name = "DPDDC-C";
                 break;
-        case MODE_I2C_READ:
-                msg[3] = 0;
-                msg_bytes = 4;
-                reply_bytes = 2;
+        case PORT_D:
+                intel_dp->aux_ch_ctl_reg = PCH_DPD_AUX_CH_CTL;
+                name = "DPDDC-D";
                 break;
         default:
-                msg_bytes = 3;
-                reply_bytes = 1;
-                break;
+                BUG();
         }
 
-        /*
-         * DP1.2 sections 2.7.7.1.5.6.1 and 2.7.7.1.6.6.1: A DP Source device is
-         * required to retry at least seven times upon receiving AUX_DEFER
-         * before giving up the AUX transaction.
-         */
-        for (retry = 0; retry < 7; retry++) {
-                ret = intel_dp_aux_ch(intel_dp,
-                                      msg, msg_bytes,
-                                      reply, reply_bytes);
-                if (ret < 0) {
-                        DRM_DEBUG_KMS("aux_ch failed %d\n", ret);
-                        goto out;
-                }
+        if (!HAS_DDI(dev))
+                intel_dp->aux_ch_ctl_reg = intel_dp->output_reg + 0x10;
 
-                switch ((reply[0] >> 4) & DP_AUX_NATIVE_REPLY_MASK) {
-                case DP_AUX_NATIVE_REPLY_ACK:
-                        /* I2C-over-AUX Reply field is only valid
-                         * when paired with AUX ACK.
-                         */
-                        break;
-                case DP_AUX_NATIVE_REPLY_NACK:
-                        DRM_DEBUG_KMS("aux_ch native nack\n");
-                        ret = -EREMOTEIO;
-                        goto out;
-                case DP_AUX_NATIVE_REPLY_DEFER:
-                        /*
-                         * For now, just give more slack to branch devices. We
-                         * could check the DPCD for I2C bit rate capabilities,
-                         * and if available, adjust the interval. We could also
-                         * be more careful with DP-to-Legacy adapters where a
-                         * long legacy cable may force very low I2C bit rates.
-                         */
-                        if (intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] &
-                            DP_DWN_STRM_PORT_PRESENT)
-                                usleep_range(500, 600);
-                        else
-                                usleep_range(300, 400);
-                        continue;
-                default:
-                        DRM_ERROR("aux_ch invalid native reply 0x%02x\n",
-                                  reply[0]);
-                        ret = -EREMOTEIO;
-                        goto out;
-                }
+        intel_dp->aux.name = name;
+        intel_dp->aux.dev = dev->dev;
+        intel_dp->aux.transfer = intel_dp_aux_transfer;
 
-                switch ((reply[0] >> 4) & DP_AUX_I2C_REPLY_MASK) {
-                case DP_AUX_I2C_REPLY_ACK:
-                        if (mode == MODE_I2C_READ) {
-                                *read_byte = reply[1];
-                        }
-                        ret = reply_bytes - 1;
-                        goto out;
-                case DP_AUX_I2C_REPLY_NACK:
-                        DRM_DEBUG_KMS("aux_i2c nack\n");
-                        ret = -EREMOTEIO;
-                        goto out;
-                case DP_AUX_I2C_REPLY_DEFER:
-                        DRM_DEBUG_KMS("aux_i2c defer\n");
-                        udelay(100);
-                        break;
-                default:
-                        DRM_ERROR("aux_i2c invalid reply 0x%02x\n", reply[0]);
-                        ret = -EREMOTEIO;
-                        goto out;
-                }
-        }
+        DRM_DEBUG_KMS("registering %s bus for %s\n", name,
+                      connector->base.kdev->kobj.name);
 
-        DRM_ERROR("too many retries, giving up\n");
-        ret = -EREMOTEIO;
+        ret = drm_dp_aux_register_i2c_bus(&intel_dp->aux);
+        if (ret < 0) {
+                DRM_ERROR("drm_dp_aux_register_i2c_bus() for %s failed (%d)\n",
+                          name, ret);
+                return;
+        }
 
-out:
-        ironlake_edp_panel_vdd_off(intel_dp, false);
-        return ret;
+        ret = sysfs_create_link(&connector->base.kdev->kobj,
+                                &intel_dp->aux.ddc.dev.kobj,
+                                intel_dp->aux.ddc.dev.kobj.name);
+        if (ret < 0) {
+                DRM_ERROR("sysfs_create_link() for %s failed (%d)\n", name, ret);
+                drm_dp_aux_unregister_i2c_bus(&intel_dp->aux);
+        }
 }
 
-static int
-intel_dp_i2c_init(struct intel_dp *intel_dp,
-                  struct intel_connector *intel_connector, const char *name)
+static void
+intel_dp_connector_unregister(struct intel_connector *intel_connector)
 {
-        int     ret;
-
-        DRM_DEBUG_KMS("i2c_init %s\n", name);
-        intel_dp->algo.running = false;
-        intel_dp->algo.address = 0;
-        intel_dp->algo.aux_ch = intel_dp_i2c_aux_ch;
-
-        memset(&intel_dp->adapter, '\0', sizeof(intel_dp->adapter));
-        intel_dp->adapter.owner = THIS_MODULE;
-        intel_dp->adapter.class = I2C_CLASS_DDC;
-        strncpy(intel_dp->adapter.name, name, sizeof(intel_dp->adapter.name) - 1);
-        intel_dp->adapter.name[sizeof(intel_dp->adapter.name) - 1] = '\0';
-        intel_dp->adapter.algo_data = &intel_dp->algo;
-        intel_dp->adapter.dev.parent = intel_connector->base.kdev;
+        struct intel_dp *intel_dp = intel_attached_dp(&intel_connector->base);
 
-        ret = i2c_dp_aux_add_bus(&intel_dp->adapter);
-        return ret;
+        sysfs_remove_link(&intel_connector->base.kdev->kobj,
+                          intel_dp->aux.ddc.dev.kobj.name);
+        intel_connector_unregister(intel_connector);
 }
 
 static void
@@ -812,9 +750,10 @@ intel_dp_compute_config(struct intel_encoder *encoder,
         struct intel_connector *intel_connector = intel_dp->attached_connector;
         int lane_count, clock;
         int max_lane_count = drm_dp_max_lane_count(intel_dp->dpcd);
-        int max_clock = intel_dp_max_link_bw(intel_dp) == DP_LINK_BW_2_7 ? 1 : 0;
+        /* Conveniently, the link BW constants become indices with a shift...*/
+        int max_clock = intel_dp_max_link_bw(intel_dp) >> 3;
         int bpp, mode_rate;
-        static int bws[2] = { DP_LINK_BW_1_62, DP_LINK_BW_2_7 };
+        static int bws[] = { DP_LINK_BW_1_62, DP_LINK_BW_2_7, DP_LINK_BW_5_4 };
         int link_avail, link_clock;
 
         if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev) && port != PORT_A)
@@ -855,8 +794,8 @@ intel_dp_compute_config(struct intel_encoder *encoder,
                 mode_rate = intel_dp_link_required(adjusted_mode->crtc_clock,
                                                    bpp);
 
-                for (clock = 0; clock <= max_clock; clock++) {
-                        for (lane_count = 1; lane_count <= max_lane_count; lane_count <<= 1) {
+                for (lane_count = 1; lane_count <= max_lane_count; lane_count <<= 1) {
+                        for (clock = 0; clock <= max_clock; clock++) {
                                 link_clock = drm_dp_bw_code_to_link_rate(bws[clock]);
                                 link_avail = intel_dp_max_data_rate(link_clock,
                                                                     lane_count);
@@ -1015,16 +954,16 @@ static void intel_dp_mode_set(struct intel_encoder *encoder)
                 ironlake_set_pll_cpu_edp(intel_dp);
 }
 
-#define IDLE_ON_MASK            (PP_ON | 0        | PP_SEQUENCE_MASK | 0                     | PP_SEQUENCE_STATE_MASK)
-#define IDLE_ON_VALUE           (PP_ON | 0        | PP_SEQUENCE_NONE | 0                     | PP_SEQUENCE_STATE_ON_IDLE)
+#define IDLE_ON_MASK            (PP_ON | PP_SEQUENCE_MASK | 0                     | PP_SEQUENCE_STATE_MASK)
+#define IDLE_ON_VALUE           (PP_ON | PP_SEQUENCE_NONE | 0                     | PP_SEQUENCE_STATE_ON_IDLE)
 
-#define IDLE_OFF_MASK           (PP_ON | 0        | PP_SEQUENCE_MASK | 0                     | PP_SEQUENCE_STATE_MASK)
-#define IDLE_OFF_VALUE          (0     | 0        | PP_SEQUENCE_NONE | 0                     | PP_SEQUENCE_STATE_OFF_IDLE)
+#define IDLE_OFF_MASK           (PP_ON | PP_SEQUENCE_MASK | 0                     | 0)
+#define IDLE_OFF_VALUE          (0     | PP_SEQUENCE_NONE | 0                     | 0)
 
-#define IDLE_CYCLE_MASK         (PP_ON | 0        | PP_SEQUENCE_MASK | PP_CYCLE_DELAY_ACTIVE | PP_SEQUENCE_STATE_MASK)
-#define IDLE_CYCLE_VALUE        (0     | 0        | PP_SEQUENCE_NONE | 0                     | PP_SEQUENCE_STATE_OFF_IDLE)
+#define IDLE_CYCLE_MASK         (PP_ON | PP_SEQUENCE_MASK | PP_CYCLE_DELAY_ACTIVE | PP_SEQUENCE_STATE_MASK)
+#define IDLE_CYCLE_VALUE        (0     | PP_SEQUENCE_NONE | 0                     | PP_SEQUENCE_STATE_OFF_IDLE)
 
-static void ironlake_wait_panel_status(struct intel_dp *intel_dp,
+static void wait_panel_status(struct intel_dp *intel_dp,
                                        u32 mask,
                                        u32 value)
 {
@@ -1049,24 +988,41 @@ static void ironlake_wait_panel_status(struct intel_dp *intel_dp,
         DRM_DEBUG_KMS("Wait complete\n");
 }
 
-static void ironlake_wait_panel_on(struct intel_dp *intel_dp)
+static void wait_panel_on(struct intel_dp *intel_dp)
 {
         DRM_DEBUG_KMS("Wait for panel power on\n");
-        ironlake_wait_panel_status(intel_dp, IDLE_ON_MASK, IDLE_ON_VALUE);
+        wait_panel_status(intel_dp, IDLE_ON_MASK, IDLE_ON_VALUE);
 }
 
-static void ironlake_wait_panel_off(struct intel_dp *intel_dp)
+static void wait_panel_off(struct intel_dp *intel_dp)
 {
         DRM_DEBUG_KMS("Wait for panel power off time\n");
-        ironlake_wait_panel_status(intel_dp, IDLE_OFF_MASK, IDLE_OFF_VALUE);
+        wait_panel_status(intel_dp, IDLE_OFF_MASK, IDLE_OFF_VALUE);
 }
 
-static void ironlake_wait_panel_power_cycle(struct intel_dp *intel_dp)
+static void wait_panel_power_cycle(struct intel_dp *intel_dp)
 {
         DRM_DEBUG_KMS("Wait for panel power cycle\n");
-        ironlake_wait_panel_status(intel_dp, IDLE_CYCLE_MASK, IDLE_CYCLE_VALUE);
+
+        /* When we disable the VDD override bit last we have to do the manual
+         * wait. */
+        wait_remaining_ms_from_jiffies(intel_dp->last_power_cycle,
+                                       intel_dp->panel_power_cycle_delay);
+
+        wait_panel_status(intel_dp, IDLE_CYCLE_MASK, IDLE_CYCLE_VALUE);
+}
+
+static void wait_backlight_on(struct intel_dp *intel_dp)
+{
+        wait_remaining_ms_from_jiffies(intel_dp->last_power_on,
+                                       intel_dp->backlight_on_delay);
 }
 
+static void edp_wait_backlight_off(struct intel_dp *intel_dp)
+{
+        wait_remaining_ms_from_jiffies(intel_dp->last_backlight_off,
+                                       intel_dp->backlight_off_delay);
+}
 
 /* Read the current pp_control value, unlocking the register if it
  * is locked
@@ -1084,30 +1040,28 @@ static  u32 ironlake_get_pp_control(struct intel_dp *intel_dp)
         return control;
 }
 
-void ironlake_edp_panel_vdd_on(struct intel_dp *intel_dp)
+static bool _edp_panel_vdd_on(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 pp;
         u32 pp_stat_reg, pp_ctrl_reg;
+        bool need_to_disable = !intel_dp->want_panel_vdd;
 
         if (!is_edp(intel_dp))
-                return;
-
-        WARN(intel_dp->want_panel_vdd,
-             "eDP VDD already requested on\n");
+                return false;
 
         intel_dp->want_panel_vdd = true;
 
-        if (ironlake_edp_have_panel_vdd(intel_dp))
-                return;
+        if (edp_have_panel_vdd(intel_dp))
+                return need_to_disable;
 
         intel_runtime_pm_get(dev_priv);
 
         DRM_DEBUG_KMS("Turning eDP VDD on\n");
 
-        if (!ironlake_edp_have_panel_power(intel_dp))
-                ironlake_wait_panel_power_cycle(intel_dp);
+        if (!edp_have_panel_power(intel_dp))
+                wait_panel_power_cycle(intel_dp);
 
         pp = ironlake_get_pp_control(intel_dp);
         pp |= EDP_FORCE_VDD;
@@ -1122,13 +1076,24 @@ void ironlake_edp_panel_vdd_on(struct intel_dp *intel_dp)
         /*
          * If the panel wasn't on, delay before accessing aux channel
          */
-        if (!ironlake_edp_have_panel_power(intel_dp)) {
+        if (!edp_have_panel_power(intel_dp)) {
                 DRM_DEBUG_KMS("eDP was not running\n");
                 msleep(intel_dp->panel_power_up_delay);
         }
+
+        return need_to_disable;
+}
+
+void intel_edp_panel_vdd_on(struct intel_dp *intel_dp)
+{
+        if (is_edp(intel_dp)) {
+                bool vdd = _edp_panel_vdd_on(intel_dp);
+
+                WARN(!vdd, "eDP VDD already requested on\n");
+        }
 }
 
-static void ironlake_panel_vdd_off_sync(struct intel_dp *intel_dp)
+static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1137,7 +1102,7 @@ static void ironlake_panel_vdd_off_sync(struct intel_dp *intel_dp)
 
         WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
 
-        if (!intel_dp->want_panel_vdd && ironlake_edp_have_panel_vdd(intel_dp)) {
+        if (!intel_dp->want_panel_vdd && edp_have_panel_vdd(intel_dp)) {
                 DRM_DEBUG_KMS("Turning eDP VDD off\n");
 
                 pp = ironlake_get_pp_control(intel_dp);
@@ -1154,24 +1119,24 @@ static void ironlake_panel_vdd_off_sync(struct intel_dp *intel_dp)
                 I915_READ(pp_stat_reg), I915_READ(pp_ctrl_reg));
 
                 if ((pp & POWER_TARGET_ON) == 0)
-                        msleep(intel_dp->panel_power_cycle_delay);
+                        intel_dp->last_power_cycle = jiffies;
 
                 intel_runtime_pm_put(dev_priv);
         }
 }
 
-static void ironlake_panel_vdd_work(struct work_struct *__work)
+static void edp_panel_vdd_work(struct work_struct *__work)
 {
         struct intel_dp *intel_dp = container_of(to_delayed_work(__work),
                                                  struct intel_dp, panel_vdd_work);
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
 
         mutex_lock(&dev->mode_config.mutex);
-        ironlake_panel_vdd_off_sync(intel_dp);
+        edp_panel_vdd_off_sync(intel_dp);
         mutex_unlock(&dev->mode_config.mutex);
 }
 
-void ironlake_edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
+static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
 {
         if (!is_edp(intel_dp))
                 return;
@@ -1181,7 +1146,7 @@ void ironlake_edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
         intel_dp->want_panel_vdd = false;
 
         if (sync) {
-                ironlake_panel_vdd_off_sync(intel_dp);
+                edp_panel_vdd_off_sync(intel_dp);
         } else {
                 /*
                  * Queue the timer to fire a long
@@ -1193,7 +1158,7 @@ void ironlake_edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
         }
 }
 
-void ironlake_edp_panel_on(struct intel_dp *intel_dp)
+void intel_edp_panel_on(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1205,12 +1170,12 @@ void ironlake_edp_panel_on(struct intel_dp *intel_dp)
 
         DRM_DEBUG_KMS("Turn eDP power on\n");
 
-        if (ironlake_edp_have_panel_power(intel_dp)) {
+        if (edp_have_panel_power(intel_dp)) {
                 DRM_DEBUG_KMS("eDP power already on\n");
                 return;
         }
 
-        ironlake_wait_panel_power_cycle(intel_dp);
+        wait_panel_power_cycle(intel_dp);
 
         pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
         pp = ironlake_get_pp_control(intel_dp);
@@ -1228,7 +1193,8 @@ void ironlake_edp_panel_on(struct intel_dp *intel_dp)
         I915_WRITE(pp_ctrl_reg, pp);
         POSTING_READ(pp_ctrl_reg);
 
-        ironlake_wait_panel_on(intel_dp);
+        wait_panel_on(intel_dp);
+        intel_dp->last_power_on = jiffies;
 
         if (IS_GEN5(dev)) {
                 pp |= PANEL_POWER_RESET; /* restore panel reset bit */
@@ -1237,7 +1203,7 @@ void ironlake_edp_panel_on(struct intel_dp *intel_dp)
         }
 }
 
-void ironlake_edp_panel_off(struct intel_dp *intel_dp)
+void intel_edp_panel_off(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1249,27 +1215,31 @@ void ironlake_edp_panel_off(struct intel_dp *intel_dp)
 
         DRM_DEBUG_KMS("Turn eDP power off\n");
 
+        edp_wait_backlight_off(intel_dp);
+
         WARN(!intel_dp->want_panel_vdd, "Need VDD to turn off panel\n");
 
         pp = ironlake_get_pp_control(intel_dp);
         /* We need to switch off panel power _and_ force vdd, for otherwise some
          * panels get very unhappy and cease to work. */
-        pp &= ~(POWER_TARGET_ON | EDP_FORCE_VDD | PANEL_POWER_RESET | EDP_BLC_ENABLE);
+        pp &= ~(POWER_TARGET_ON | PANEL_POWER_RESET | EDP_FORCE_VDD |
+                EDP_BLC_ENABLE);
 
         pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
 
+        intel_dp->want_panel_vdd = false;
+
         I915_WRITE(pp_ctrl_reg, pp);
         POSTING_READ(pp_ctrl_reg);
 
-        intel_dp->want_panel_vdd = false;
-
-        ironlake_wait_panel_off(intel_dp);
+        intel_dp->last_power_cycle = jiffies;
+        wait_panel_off(intel_dp);
 
         /* We got a reference when we enabled the VDD. */
         intel_runtime_pm_put(dev_priv);
 }
 
-void ironlake_edp_backlight_on(struct intel_dp *intel_dp)
+void intel_edp_backlight_on(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
@@ -1287,7 +1257,7 @@ void ironlake_edp_backlight_on(struct intel_dp *intel_dp)
          * link.  So delay a bit to make sure the image is solid before
          * allowing it to appear.
          */
-        msleep(intel_dp->backlight_on_delay);
+        wait_backlight_on(intel_dp);
         pp = ironlake_get_pp_control(intel_dp);
         pp |= EDP_BLC_ENABLE;
 
@@ -1299,7 +1269,7 @@ void ironlake_edp_backlight_on(struct intel_dp *intel_dp)
         intel_panel_enable_backlight(intel_dp->attached_connector);
 }
 
-void ironlake_edp_backlight_off(struct intel_dp *intel_dp)
+void intel_edp_backlight_off(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1319,7 +1289,7 @@ void ironlake_edp_backlight_off(struct intel_dp *intel_dp)
 
         I915_WRITE(pp_ctrl_reg, pp);
         POSTING_READ(pp_ctrl_reg);
-        msleep(intel_dp->backlight_off_delay);
+        intel_dp->last_backlight_off = jiffies;
 }
 
 static void ironlake_edp_pll_on(struct intel_dp *intel_dp)
@@ -1383,8 +1353,8 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode)
                 return;
 
         if (mode != DRM_MODE_DPMS_ON) {
-                ret = intel_dp_aux_native_write_1(intel_dp, DP_SET_POWER,
-                                                  DP_SET_POWER_D3);
+                ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER,
+                                         DP_SET_POWER_D3);
                 if (ret != 1)
                         DRM_DEBUG_DRIVER("failed to write sink power state\n");
         } else {
@@ -1393,9 +1363,8 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode)
                  * time to wake up.
                  */
                 for (i = 0; i < 3; i++) {
-                        ret = intel_dp_aux_native_write_1(intel_dp,
-                                                          DP_SET_POWER,
-                                                          DP_SET_POWER_D0);
+                        ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER,
+                                                 DP_SET_POWER_D0);
                         if (ret == 1)
                                 break;
                         msleep(1);
@@ -1410,7 +1379,14 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
         enum port port = dp_to_dig_port(intel_dp)->port;
         struct drm_device *dev = encoder->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 tmp = I915_READ(intel_dp->output_reg);
+        enum intel_display_power_domain power_domain;
+        u32 tmp;
+
+        power_domain = intel_display_port_power_domain(encoder);
+        if (!intel_display_power_enabled(dev_priv, power_domain))
+                return false;
+
+        tmp = I915_READ(intel_dp->output_reg);
 
         if (!(tmp & DP_PORT_EN))
                 return false;
@@ -1604,19 +1580,19 @@ static void intel_edp_psr_enable_sink(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct drm_i915_private *dev_priv = dev->dev_private;
-        uint32_t aux_clock_divider = get_aux_clock_divider(intel_dp, 0);
+        uint32_t aux_clock_divider;
         int precharge = 0x3;
         int msg_size = 5;       /* Header(4) + Message(1) */
 
+        aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, 0);
+
         /* Enable PSR in sink */
         if (intel_dp->psr_dpcd[1] & DP_PSR_NO_TRAIN_ON_EXIT)
-                intel_dp_aux_native_write_1(intel_dp, DP_PSR_EN_CFG,
-                                            DP_PSR_ENABLE &
-                                            ~DP_PSR_MAIN_LINK_ACTIVE);
+                drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG,
+                                   DP_PSR_ENABLE & ~DP_PSR_MAIN_LINK_ACTIVE);
         else
-                intel_dp_aux_native_write_1(intel_dp, DP_PSR_EN_CFG,
-                                            DP_PSR_ENABLE |
-                                            DP_PSR_MAIN_LINK_ACTIVE);
+                drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG,
+                                   DP_PSR_ENABLE | DP_PSR_MAIN_LINK_ACTIVE);
 
         /* Setup AUX registers */
         I915_WRITE(EDP_PSR_AUX_DATA1(dev), EDP_PSR_DPCD_COMMAND);
@@ -1659,7 +1635,7 @@ static bool intel_edp_psr_match_conditions(struct intel_dp *intel_dp)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_crtc *crtc = dig_port->base.base.crtc;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct drm_i915_gem_object *obj = to_intel_framebuffer(crtc->fb)->obj;
+        struct drm_i915_gem_object *obj = to_intel_framebuffer(crtc->primary->fb)->obj;
         struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base;
 
         dev_priv->psr.source_ok = false;
@@ -1675,7 +1651,7 @@ static bool intel_edp_psr_match_conditions(struct intel_dp *intel_dp)
                 return false;
         }
 
-        if (!i915_enable_psr) {
+        if (!i915.enable_psr) {
                 DRM_DEBUG_KMS("PSR disable by flag\n");
                 return false;
         }
@@ -1692,7 +1668,7 @@ static bool intel_edp_psr_match_conditions(struct intel_dp *intel_dp)
                 return false;
         }
 
-        obj = to_intel_framebuffer(crtc->fb)->obj;
+        obj = to_intel_framebuffer(crtc->primary->fb)->obj;
         if (obj->tiling_mode != I915_TILING_X ||
             obj->fence_reg == I915_FENCE_REG_NONE) {
                 DRM_DEBUG_KMS("PSR condition failed: fb not tiled or fenced\n");
@@ -1791,10 +1767,10 @@ static void intel_disable_dp(struct intel_encoder *encoder)
 
         /* Make sure the panel is off before trying to change the mode. But also
          * ensure that we have vdd while we switch off the panel. */
-        ironlake_edp_panel_vdd_on(intel_dp);
-        ironlake_edp_backlight_off(intel_dp);
+        intel_edp_panel_vdd_on(intel_dp);
+        intel_edp_backlight_off(intel_dp);
         intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
-        ironlake_edp_panel_off(intel_dp);
+        intel_edp_panel_off(intel_dp);
 
         /* cpu edp my only be disable _after_ the cpu pipe/plane is disabled. */
         if (!(port == PORT_A || IS_VALLEYVIEW(dev)))
@@ -1824,11 +1800,11 @@ static void intel_enable_dp(struct intel_encoder *encoder)
         if (WARN_ON(dp_reg & DP_PORT_EN))
                 return;
 
-        ironlake_edp_panel_vdd_on(intel_dp);
+        intel_edp_panel_vdd_on(intel_dp);
         intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
         intel_dp_start_link_train(intel_dp);
-        ironlake_edp_panel_on(intel_dp);
-        ironlake_edp_panel_vdd_off(intel_dp, true);
+        intel_edp_panel_on(intel_dp);
+        edp_panel_vdd_off(intel_dp, true);
         intel_dp_complete_link_train(intel_dp);
         intel_dp_stop_link_train(intel_dp);
 }
@@ -1838,14 +1814,14 @@ static void g4x_enable_dp(struct intel_encoder *encoder)
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
         intel_enable_dp(encoder);
-        ironlake_edp_backlight_on(intel_dp);
+        intel_edp_backlight_on(intel_dp);
 }
 
 static void vlv_enable_dp(struct intel_encoder *encoder)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
-        ironlake_edp_backlight_on(intel_dp);
+        intel_edp_backlight_on(intel_dp);
 }
 
 static void g4x_pre_enable_dp(struct intel_encoder *encoder)
@@ -1927,26 +1903,25 @@ static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder)
 /*
  * Native read with retry for link status and receiver capability reads for
  * cases where the sink may still be asleep.
+ *
+ * Sinks are *supposed* to come up within 1ms from an off state, but we're also
+ * supposed to retry 3 times per the spec.
  */
-static bool
-intel_dp_aux_native_read_retry(struct intel_dp *intel_dp, uint16_t address,
-                               uint8_t *recv, int recv_bytes)
+static ssize_t
+intel_dp_dpcd_read_wake(struct drm_dp_aux *aux, unsigned int offset,
+                        void *buffer, size_t size)
 {
-        int ret, i;
+        ssize_t ret;
+        int i;
 
-        /*
-         * Sinks are *supposed* to come up within 1ms from an off state,
-         * but we're also supposed to retry 3 times per the spec.
-         */
         for (i = 0; i < 3; i++) {
-                ret = intel_dp_aux_native_read(intel_dp, address, recv,
-                                               recv_bytes);
-                if (ret == recv_bytes)
-                        return true;
+                ret = drm_dp_dpcd_read(aux, offset, buffer, size);
+                if (ret == size)
+                        return ret;
                 msleep(1);
         }
 
-        return false;
+        return ret;
 }
 
 /*
@@ -1956,10 +1931,10 @@ intel_dp_aux_native_read_retry(struct intel_dp *intel_dp, uint16_t address,
 static bool
 intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_STATUS_SIZE])
 {
-        return intel_dp_aux_native_read_retry(intel_dp,
-                                              DP_LANE0_1_STATUS,
-                                              link_status,
-                                              DP_LINK_STATUS_SIZE);
+        return intel_dp_dpcd_read_wake(&intel_dp->aux,
+                                       DP_LANE0_1_STATUS,
+                                       link_status,
+                                       DP_LINK_STATUS_SIZE) == DP_LINK_STATUS_SIZE;
 }
 
 /*
@@ -2473,8 +2448,8 @@ intel_dp_set_link_train(struct intel_dp *intel_dp,
                 len = intel_dp->lane_count + 1;
         }
 
-        ret = intel_dp_aux_native_write(intel_dp, DP_TRAINING_PATTERN_SET,
-                                        buf, len);
+        ret = drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_PATTERN_SET,
+                                buf, len);
 
         return ret == len;
 }
@@ -2503,9 +2478,8 @@ intel_dp_update_link_train(struct intel_dp *intel_dp, uint32_t *DP,
         I915_WRITE(intel_dp->output_reg, *DP);
         POSTING_READ(intel_dp->output_reg);
 
-        ret = intel_dp_aux_native_write(intel_dp, DP_TRAINING_LANE0_SET,
-                                        intel_dp->train_set,
-                                        intel_dp->lane_count);
+        ret = drm_dp_dpcd_write(&intel_dp->aux, DP_TRAINING_LANE0_SET,
+                                intel_dp->train_set, intel_dp->lane_count);
 
         return ret == intel_dp->lane_count;
 }
@@ -2561,11 +2535,11 @@ intel_dp_start_link_train(struct intel_dp *intel_dp)
         link_config[1] = intel_dp->lane_count;
         if (drm_dp_enhanced_frame_cap(intel_dp->dpcd))
                 link_config[1] |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
-        intel_dp_aux_native_write(intel_dp, DP_LINK_BW_SET, link_config, 2);
+        drm_dp_dpcd_write(&intel_dp->aux, DP_LINK_BW_SET, link_config, 2);
 
         link_config[0] = 0;
         link_config[1] = DP_SET_ANSI_8B10B;
-        intel_dp_aux_native_write(intel_dp, DP_DOWNSPREAD_CTRL, link_config, 2);
+        drm_dp_dpcd_write(&intel_dp->aux, DP_DOWNSPREAD_CTRL, link_config, 2);
 
         DP |= DP_PORT_EN;
 
@@ -2638,10 +2612,15 @@ intel_dp_complete_link_train(struct intel_dp *intel_dp)
         bool channel_eq = false;
         int tries, cr_tries;
         uint32_t DP = intel_dp->DP;
+        uint32_t training_pattern = DP_TRAINING_PATTERN_2;
+
+        /* Training Pattern 3 for HBR2 ot 1.2 devices that support it*/
+        if (intel_dp->link_bw == DP_LINK_BW_5_4 || intel_dp->use_tps3)
+                training_pattern = DP_TRAINING_PATTERN_3;
 
         /* channel equalization */
         if (!intel_dp_set_link_train(intel_dp, &DP,
-                                     DP_TRAINING_PATTERN_2 |
+                                     training_pattern |
                                      DP_LINK_SCRAMBLING_DISABLE)) {
                 DRM_ERROR("failed to start channel equalization\n");
                 return;
@@ -2668,7 +2647,7 @@ intel_dp_complete_link_train(struct intel_dp *intel_dp)
                 if (!drm_dp_clock_recovery_ok(link_status, intel_dp->lane_count)) {
                         intel_dp_start_link_train(intel_dp);
                         intel_dp_set_link_train(intel_dp, &DP,
-                                                DP_TRAINING_PATTERN_2 |
+                                                training_pattern |
                                                 DP_LINK_SCRAMBLING_DISABLE);
                         cr_tries++;
                         continue;
@@ -2684,7 +2663,7 @@ intel_dp_complete_link_train(struct intel_dp *intel_dp)
                         intel_dp_link_down(intel_dp);
                         intel_dp_start_link_train(intel_dp);
                         intel_dp_set_link_train(intel_dp, &DP,
-                                                DP_TRAINING_PATTERN_2 |
+                                                training_pattern |
                                                 DP_LINK_SCRAMBLING_DISABLE);
                         tries = 0;
                         cr_tries++;
@@ -2803,8 +2782,8 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
 
         char dpcd_hex_dump[sizeof(intel_dp->dpcd) * 3];
 
-        if (intel_dp_aux_native_read_retry(intel_dp, 0x000, intel_dp->dpcd,
-                                           sizeof(intel_dp->dpcd)) == 0)
+        if (intel_dp_dpcd_read_wake(&intel_dp->aux, 0x000, intel_dp->dpcd,
+                                    sizeof(intel_dp->dpcd)) < 0)
                 return false; /* aux transfer failed */
 
         hex_dump_to_buffer(intel_dp->dpcd, sizeof(intel_dp->dpcd),
@@ -2817,15 +2796,23 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
         /* Check if the panel supports PSR */
         memset(intel_dp->psr_dpcd, 0, sizeof(intel_dp->psr_dpcd));
         if (is_edp(intel_dp)) {
-                intel_dp_aux_native_read_retry(intel_dp, DP_PSR_SUPPORT,
-                                               intel_dp->psr_dpcd,
-                                               sizeof(intel_dp->psr_dpcd));
+                intel_dp_dpcd_read_wake(&intel_dp->aux, DP_PSR_SUPPORT,
+                                        intel_dp->psr_dpcd,
+                                        sizeof(intel_dp->psr_dpcd));
                 if (intel_dp->psr_dpcd[0] & DP_PSR_IS_SUPPORTED) {
                         dev_priv->psr.sink_support = true;
                         DRM_DEBUG_KMS("Detected EDP PSR Panel.\n");
                 }
         }
 
+        /* Training Pattern 3 support */
+        if (intel_dp->dpcd[DP_DPCD_REV] >= 0x12 &&
+            intel_dp->dpcd[DP_MAX_LANE_COUNT] & DP_TPS3_SUPPORTED) {
+                intel_dp->use_tps3 = true;
+                DRM_DEBUG_KMS("Displayport TPS3 supported");
+        } else
+                intel_dp->use_tps3 = false;
+
         if (!(intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] &
               DP_DWN_STRM_PORT_PRESENT))
                 return true; /* native DP sink */
@@ -2833,9 +2820,9 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
         if (intel_dp->dpcd[DP_DPCD_REV] == 0x10)
                 return true; /* no per-port downstream info */
 
-        if (intel_dp_aux_native_read_retry(intel_dp, DP_DOWNSTREAM_PORT_0,
-                                           intel_dp->downstream_ports,
-                                           DP_MAX_DOWNSTREAM_PORTS) == 0)
+        if (intel_dp_dpcd_read_wake(&intel_dp->aux, DP_DOWNSTREAM_PORT_0,
+                                    intel_dp->downstream_ports,
+                                    DP_MAX_DOWNSTREAM_PORTS) < 0)
                 return false; /* downstream port status fetch failed */
 
         return true;
@@ -2849,38 +2836,61 @@ intel_dp_probe_oui(struct intel_dp *intel_dp)
         if (!(intel_dp->dpcd[DP_DOWN_STREAM_PORT_COUNT] & DP_OUI_SUPPORT))
                 return;
 
-        ironlake_edp_panel_vdd_on(intel_dp);
+        intel_edp_panel_vdd_on(intel_dp);
 
-        if (intel_dp_aux_native_read_retry(intel_dp, DP_SINK_OUI, buf, 3))
+        if (intel_dp_dpcd_read_wake(&intel_dp->aux, DP_SINK_OUI, buf, 3) == 3)
                 DRM_DEBUG_KMS("Sink OUI: %02hx%02hx%02hx\n",
                               buf[0], buf[1], buf[2]);
 
-        if (intel_dp_aux_native_read_retry(intel_dp, DP_BRANCH_OUI, buf, 3))
+        if (intel_dp_dpcd_read_wake(&intel_dp->aux, DP_BRANCH_OUI, buf, 3) == 3)
                 DRM_DEBUG_KMS("Branch OUI: %02hx%02hx%02hx\n",
                               buf[0], buf[1], buf[2]);
 
-        ironlake_edp_panel_vdd_off(intel_dp, false);
+        edp_panel_vdd_off(intel_dp, false);
 }
 
-static bool
-intel_dp_get_sink_irq(struct intel_dp *intel_dp, u8 *sink_irq_vector)
+int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc)
 {
-        int ret;
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct drm_device *dev = intel_dig_port->base.base.dev;
+        struct intel_crtc *intel_crtc =
+                to_intel_crtc(intel_dig_port->base.base.crtc);
+        u8 buf[1];
 
-        ret = intel_dp_aux_native_read_retry(intel_dp,
-                                             DP_DEVICE_SERVICE_IRQ_VECTOR,
-                                             sink_irq_vector, 1);
-        if (!ret)
-                return false;
+        if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, buf) < 0)
+                return -EAGAIN;
 
-        return true;
+        if (!(buf[0] & DP_TEST_CRC_SUPPORTED))
+                return -ENOTTY;
+
+        if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK,
+                               DP_TEST_SINK_START) < 0)
+                return -EAGAIN;
+
+        /* Wait 2 vblanks to be sure we will have the correct CRC value */
+        intel_wait_for_vblank(dev, intel_crtc->pipe);
+        intel_wait_for_vblank(dev, intel_crtc->pipe);
+
+        if (drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_CRC_R_CR, crc, 6) < 0)
+                return -EAGAIN;
+
+        drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK, 0);
+        return 0;
+}
+
+static bool
+intel_dp_get_sink_irq(struct intel_dp *intel_dp, u8 *sink_irq_vector)
+{
+        return intel_dp_dpcd_read_wake(&intel_dp->aux,
+                                       DP_DEVICE_SERVICE_IRQ_VECTOR,
+                                       sink_irq_vector, 1) == 1;
 }
 
 static void
 intel_dp_handle_test_request(struct intel_dp *intel_dp)
 {
         /* NAK by default */
-        intel_dp_aux_native_write_1(intel_dp, DP_TEST_RESPONSE, DP_TEST_NAK);
+        drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_RESPONSE, DP_TEST_NAK);
 }
 
 /*
@@ -2919,9 +2929,9 @@ intel_dp_check_link_status(struct intel_dp *intel_dp)
         if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11 &&
             intel_dp_get_sink_irq(intel_dp, &sink_irq_vector)) {
                 /* Clear interrupt source */
-                intel_dp_aux_native_write_1(intel_dp,
-                                            DP_DEVICE_SERVICE_IRQ_VECTOR,
-                                            sink_irq_vector);
+                drm_dp_dpcd_writeb(&intel_dp->aux,
+                                   DP_DEVICE_SERVICE_IRQ_VECTOR,
+                                   sink_irq_vector);
 
                 if (sink_irq_vector & DP_AUTOMATED_TEST_REQUEST)
                         intel_dp_handle_test_request(intel_dp);
@@ -2956,15 +2966,17 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp)
         if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11 &&
             intel_dp->downstream_ports[0] & DP_DS_PORT_HPD) {
                 uint8_t reg;
-                if (!intel_dp_aux_native_read_retry(intel_dp, DP_SINK_COUNT,
-                                                    &reg, 1))
+
+                if (intel_dp_dpcd_read_wake(&intel_dp->aux, DP_SINK_COUNT,
+                                            &reg, 1) < 0)
                         return connector_status_unknown;
+
                 return DP_GET_SINK_COUNT(reg) ? connector_status_connected
                                               : connector_status_disconnected;
         }
 
         /* If no HPD, poke DDC gently */
-        if (drm_probe_ddc(&intel_dp->adapter))
+        if (drm_probe_ddc(&intel_dp->aux.ddc))
                 return connector_status_connected;
 
         /* Well we tried, say unknown for unreliable port types */
@@ -3106,10 +3118,14 @@ intel_dp_detect(struct drm_connector *connector, bool force)
         struct drm_device *dev = connector->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         enum drm_connector_status status;
+        enum intel_display_power_domain power_domain;
         struct edid *edid = NULL;
 
         intel_runtime_pm_get(dev_priv);
 
+        power_domain = intel_display_port_power_domain(intel_encoder);
+        intel_display_power_get(dev_priv, power_domain);
+
         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
                       connector->base.id, drm_get_connector_name(connector));
 
@@ -3128,7 +3144,7 @@ intel_dp_detect(struct drm_connector *connector, bool force)
         if (intel_dp->force_audio != HDMI_AUDIO_AUTO) {
                 intel_dp->has_audio = (intel_dp->force_audio == HDMI_AUDIO_ON);
         } else {
-                edid = intel_dp_get_edid(connector, &intel_dp->adapter);
+                edid = intel_dp_get_edid(connector, &intel_dp->aux.ddc);
                 if (edid) {
                         intel_dp->has_audio = drm_detect_monitor_audio(edid);
                         kfree(edid);
@@ -3140,21 +3156,32 @@ intel_dp_detect(struct drm_connector *connector, bool force)
         status = connector_status_connected;
 
 out:
+        intel_display_power_put(dev_priv, power_domain);
+
         intel_runtime_pm_put(dev_priv);
+
         return status;
 }
 
 static int intel_dp_get_modes(struct drm_connector *connector)
 {
         struct intel_dp *intel_dp = intel_attached_dp(connector);
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct intel_encoder *intel_encoder = &intel_dig_port->base;
         struct intel_connector *intel_connector = to_intel_connector(connector);
         struct drm_device *dev = connector->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum intel_display_power_domain power_domain;
         int ret;
 
         /* We should parse the EDID data and find out if it has an audio sink
          */
 
-        ret = intel_dp_get_edid_modes(connector, &intel_dp->adapter);
+        power_domain = intel_display_port_power_domain(intel_encoder);
+        intel_display_power_get(dev_priv, power_domain);
+
+        ret = intel_dp_get_edid_modes(connector, &intel_dp->aux.ddc);
+        intel_display_power_put(dev_priv, power_domain);
         if (ret)
                 return ret;
 
@@ -3175,15 +3202,25 @@ static bool
 intel_dp_detect_audio(struct drm_connector *connector)
 {
         struct intel_dp *intel_dp = intel_attached_dp(connector);
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct intel_encoder *intel_encoder = &intel_dig_port->base;
+        struct drm_device *dev = connector->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum intel_display_power_domain power_domain;
         struct edid *edid;
         bool has_audio = false;
 
-        edid = intel_dp_get_edid(connector, &intel_dp->adapter);
+        power_domain = intel_display_port_power_domain(intel_encoder);
+        intel_display_power_get(dev_priv, power_domain);
+
+        edid = intel_dp_get_edid(connector, &intel_dp->aux.ddc);
         if (edid) {
                 has_audio = drm_detect_monitor_audio(edid);
                 kfree(edid);
         }
 
+        intel_display_power_put(dev_priv, power_domain);
+
         return has_audio;
 }
 
@@ -3298,12 +3335,12 @@ void intel_dp_encoder_destroy(struct drm_encoder *encoder)
         struct intel_dp *intel_dp = &intel_dig_port->dp;
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
 
-        i2c_del_adapter(&intel_dp->adapter);
+        drm_dp_aux_unregister_i2c_bus(&intel_dp->aux);
         drm_encoder_cleanup(encoder);
         if (is_edp(intel_dp)) {
                 cancel_delayed_work_sync(&intel_dp->panel_vdd_work);
                 mutex_lock(&dev->mode_config.mutex);
-                ironlake_panel_vdd_off_sync(intel_dp);
+                edp_panel_vdd_off_sync(intel_dp);
                 mutex_unlock(&dev->mode_config.mutex);
         }
         kfree(intel_dig_port);
@@ -3402,6 +3439,13 @@ intel_dp_add_properties(struct intel_dp *intel_dp, struct drm_connector *connect
         }
 }
 
+static void intel_dp_init_panel_power_timestamps(struct intel_dp *intel_dp)
+{
+        intel_dp->last_power_cycle = jiffies;
+        intel_dp->last_power_on = jiffies;
+        intel_dp->last_backlight_off = jiffies;
+}
+
 static void
 intel_dp_init_panel_power_sequencer(struct drm_device *dev,
                                     struct intel_dp *intel_dp,
@@ -3524,10 +3568,17 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
                 pp_div_reg = VLV_PIPE_PP_DIVISOR(pipe);
         }
 
-        /* And finally store the new values in the power sequencer. */
+        /*
+         * And finally store the new values in the power sequencer. The
+         * backlight delays are set to 1 because we do manual waits on them. For
+         * T8, even BSpec recommends doing it. For T9, if we don't do this,
+         * we'll end up waiting for the backlight off delay twice: once when we
+         * do the manual sleep, and once when we disable the panel and wait for
+         * the PP_STATUS bit to become zero.
+         */
         pp_on = (seq->t1_t3 << PANEL_POWER_UP_DELAY_SHIFT) |
-                (seq->t8 << PANEL_LIGHT_ON_DELAY_SHIFT);
-        pp_off = (seq->t9 << PANEL_LIGHT_OFF_DELAY_SHIFT) |
+                (1 << PANEL_LIGHT_ON_DELAY_SHIFT);
+        pp_off = (1 << PANEL_LIGHT_OFF_DELAY_SHIFT) |
                  (seq->t10 << PANEL_POWER_DOWN_DELAY_SHIFT);
         /* Compute the divisor for the pp clock, simply match the Bspec
          * formula. */
@@ -3562,14 +3613,14 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 }
 
 static bool intel_edp_init_connector(struct intel_dp *intel_dp,
-                                     struct intel_connector *intel_connector)
+                                     struct intel_connector *intel_connector,
+                                     struct edp_power_seq *power_seq)
 {
         struct drm_connector *connector = &intel_connector->base;
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_display_mode *fixed_mode = NULL;
-        struct edp_power_seq power_seq = { 0 };
         bool has_dpcd;
         struct drm_display_mode *scan;
         struct edid *edid;
@@ -3577,12 +3628,10 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
         if (!is_edp(intel_dp))
                 return true;
 
-        intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
-
         /* Cache DPCD and EDID for edp. */
-        ironlake_edp_panel_vdd_on(intel_dp);
+        intel_edp_panel_vdd_on(intel_dp);
         has_dpcd = intel_dp_get_dpcd(intel_dp);
-        ironlake_edp_panel_vdd_off(intel_dp, false);
+        edp_panel_vdd_off(intel_dp, false);
 
         if (has_dpcd) {
                 if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11)
@@ -3596,10 +3645,10 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
         }
 
         /* We now know it's not a ghost, init power sequence regs. */
-        intel_dp_init_panel_power_sequencer_registers(dev, intel_dp,
-                                                      &power_seq);
+        intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, power_seq);
 
-        edid = drm_get_edid(connector, &intel_dp->adapter);
+        mutex_lock(&dev->mode_config.mutex);
+        edid = drm_get_edid(connector, &intel_dp->aux.ddc);
         if (edid) {
                 if (drm_add_edid_modes(connector, edid)) {
                         drm_mode_connector_update_edid_property(connector,
@@ -3629,8 +3678,9 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
                 if (fixed_mode)
                         fixed_mode->type |= DRM_MODE_TYPE_PREFERRED;
         }
+        mutex_unlock(&dev->mode_config.mutex);
 
-        intel_panel_init(&intel_connector->panel, fixed_mode);
+        intel_panel_init(&intel_connector->panel, fixed_mode, NULL);
         intel_panel_setup_backlight(connector);
 
         return true;
@@ -3646,8 +3696,20 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
         struct drm_device *dev = intel_encoder->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         enum port port = intel_dig_port->port;
-        const char *name = NULL;
-        int type, error;
+        struct edp_power_seq power_seq = { 0 };
+        int type;
+
+        /* intel_dp vfuncs */
+        if (IS_VALLEYVIEW(dev))
+                intel_dp->get_aux_clock_divider = vlv_get_aux_clock_divider;
+        else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+                intel_dp->get_aux_clock_divider = hsw_get_aux_clock_divider;
+        else if (HAS_PCH_SPLIT(dev))
+                intel_dp->get_aux_clock_divider = ilk_get_aux_clock_divider;
+        else
+                intel_dp->get_aux_clock_divider = i9xx_get_aux_clock_divider;
+
+        intel_dp->get_aux_send_ctl = i9xx_get_aux_send_ctl;
 
         /* Preserve the current hw state. */
         intel_dp->DP = I915_READ(intel_dp->output_reg);
@@ -3677,7 +3739,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
         connector->doublescan_allowed = 0;
 
         INIT_DELAYED_WORK(&intel_dp->panel_vdd_work,
-                          ironlake_panel_vdd_work);
+                          edp_panel_vdd_work);
 
         intel_connector_attach_encoder(intel_connector, intel_encoder);
         drm_sysfs_connector_add(connector);
@@ -3686,61 +3748,41 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
                 intel_connector->get_hw_state = intel_ddi_connector_get_hw_state;
         else
                 intel_connector->get_hw_state = intel_connector_get_hw_state;
+        intel_connector->unregister = intel_dp_connector_unregister;
 
-        intel_dp->aux_ch_ctl_reg = intel_dp->output_reg + 0x10;
-        if (HAS_DDI(dev)) {
-                switch (intel_dig_port->port) {
-                case PORT_A:
-                        intel_dp->aux_ch_ctl_reg = DPA_AUX_CH_CTL;
-                        break;
-                case PORT_B:
-                        intel_dp->aux_ch_ctl_reg = PCH_DPB_AUX_CH_CTL;
-                        break;
-                case PORT_C:
-                        intel_dp->aux_ch_ctl_reg = PCH_DPC_AUX_CH_CTL;
-                        break;
-                case PORT_D:
-                        intel_dp->aux_ch_ctl_reg = PCH_DPD_AUX_CH_CTL;
-                        break;
-                default:
-                        BUG();
-                }
-        }
-
-        /* Set up the DDC bus. */
+        /* Set up the hotplug pin. */
         switch (port) {
         case PORT_A:
                 intel_encoder->hpd_pin = HPD_PORT_A;
-                name = "DPDDC-A";
                 break;
         case PORT_B:
                 intel_encoder->hpd_pin = HPD_PORT_B;
-                name = "DPDDC-B";
                 break;
         case PORT_C:
                 intel_encoder->hpd_pin = HPD_PORT_C;
-                name = "DPDDC-C";
                 break;
         case PORT_D:
                 intel_encoder->hpd_pin = HPD_PORT_D;
-                name = "DPDDC-D";
                 break;
         default:
                 BUG();
         }
 
-        error = intel_dp_i2c_init(intel_dp, intel_connector, name);
-        WARN(error, "intel_dp_i2c_init failed with error %d for port %c\n",
-             error, port_name(port));
+        if (is_edp(intel_dp)) {
+                intel_dp_init_panel_power_timestamps(intel_dp);
+                intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
+        }
+
+        intel_dp_aux_init(intel_dp, intel_connector);
 
         intel_dp->psr_setup_done = false;
 
-        if (!intel_edp_init_connector(intel_dp, intel_connector)) {
-                i2c_del_adapter(&intel_dp->adapter);
+        if (!intel_edp_init_connector(intel_dp, intel_connector, &power_seq)) {
+                drm_dp_aux_unregister_i2c_bus(&intel_dp->aux);
                 if (is_edp(intel_dp)) {
                         cancel_delayed_work_sync(&intel_dp->panel_vdd_work);
                         mutex_lock(&dev->mode_config.mutex);
-                        ironlake_panel_vdd_off_sync(intel_dp);
+                        edp_panel_vdd_off_sync(intel_dp);
                         mutex_unlock(&dev->mode_config.mutex);
                 }
                 drm_sysfs_connector_remove(connector);
@@ -3806,7 +3848,7 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port)
 
         intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT;
         intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
-        intel_encoder->cloneable = false;
+        intel_encoder->cloneable = 0;
         intel_encoder->hot_plug = intel_dp_hot_plug;
 
         if (!intel_dp_init_connector(intel_dig_port, intel_connector)) {
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index fbfaaba5cc3b..0542de982260 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -78,6 +78,12 @@
 #define MAX_OUTPUTS 6
 /* maximum connectors per crtcs in the mode set */
 
+/* Maximum cursor sizes */
+#define GEN2_CURSOR_WIDTH 64
+#define GEN2_CURSOR_HEIGHT 64
+#define CURSOR_WIDTH 256
+#define CURSOR_HEIGHT 256
+
 #define INTEL_I2C_BUS_DVO 1
 #define INTEL_I2C_BUS_SDVO 2
 
@@ -110,9 +116,10 @@ struct intel_framebuffer {
 
 struct intel_fbdev {
         struct drm_fb_helper helper;
-        struct intel_framebuffer ifb;
+        struct intel_framebuffer *fb;
         struct list_head fbdev_list;
         struct drm_display_mode *our_mode;
+        int preferred_bpp;
 };
 
 struct intel_encoder {
@@ -124,11 +131,7 @@ struct intel_encoder {
         struct intel_crtc *new_crtc;
 
         int type;
-        /*
-         * Intel hw has only one MUX where encoders could be clone, hence a
-         * simple flag is enough to compute the possible_clones mask.
-         */
-        bool cloneable;
+        unsigned int cloneable;
         bool connectors_active;
         void (*hot_plug)(struct intel_encoder *);
         bool (*compute_config)(struct intel_encoder *,
@@ -187,6 +190,14 @@ struct intel_connector {
          * and active (i.e. dpms ON state). */
         bool (*get_hw_state)(struct intel_connector *);
 
+        /*
+         * Removes all interfaces through which the connector is accessible
+         * - like sysfs, debugfs entries -, so that no new operations can be
+         * started on the connector. Also makes sure all currently pending
+         * operations finish before returing.
+         */
+        void (*unregister)(struct intel_connector *);
+
         /* Panel info for eDP and LVDS */
         struct intel_panel panel;
 
@@ -210,6 +221,12 @@ typedef struct dpll {
         int     p;
 } intel_clock_t;
 
+struct intel_plane_config {
+        bool tiled;
+        int size;
+        u32 base;
+};
+
 struct intel_crtc_config {
         /**
          * quirks - bitfield with hw state readout quirks
@@ -356,9 +373,13 @@ struct intel_crtc {
         uint32_t cursor_addr;
         int16_t cursor_x, cursor_y;
         int16_t cursor_width, cursor_height;
+        int16_t max_cursor_width, max_cursor_height;
         bool cursor_visible;
 
+        struct intel_plane_config plane_config;
         struct intel_crtc_config config;
+        struct intel_crtc_config *new_config;
+        bool new_enabled;
 
         uint32_t ddi_pll_sel;
 
@@ -475,8 +496,7 @@ struct intel_dp {
         uint8_t dpcd[DP_RECEIVER_CAP_SIZE];
         uint8_t psr_dpcd[EDP_PSR_RECEIVER_CAP_SIZE];
         uint8_t downstream_ports[DP_MAX_DOWNSTREAM_PORTS];
-        struct i2c_adapter adapter;
-        struct i2c_algo_dp_aux_data algo;
+        struct drm_dp_aux aux;
         uint8_t train_set[4];
         int panel_power_up_delay;
         int panel_power_down_delay;
@@ -485,8 +505,22 @@ struct intel_dp {
         int backlight_off_delay;
         struct delayed_work panel_vdd_work;
         bool want_panel_vdd;
+        unsigned long last_power_cycle;
+        unsigned long last_power_on;
+        unsigned long last_backlight_off;
         bool psr_setup_done;
+        bool use_tps3;
         struct intel_connector *attached_connector;
+
+        uint32_t (*get_aux_clock_divider)(struct intel_dp *dp, int index);
+        /*
+         * This function returns the value we have to program the AUX_CTL
+         * register with to kick off an AUX transaction.
+         */
+        uint32_t (*get_aux_send_ctl)(struct intel_dp *dp,
+                                     bool has_aux_irq,
+                                     int send_bytes,
+                                     uint32_t aux_clock_divider);
 };
 
 struct intel_digital_port {
@@ -540,6 +574,7 @@ struct intel_unpin_work {
 struct intel_set_config {
         struct drm_encoder **save_connector_encoders;
         struct drm_crtc **save_encoder_crtcs;
+        bool *save_crtc_enabled;
 
         bool fb_changed;
         bool mode_changed;
@@ -584,6 +619,8 @@ hdmi_to_dig_port(struct intel_hdmi *intel_hdmi)
 /* i915_irq.c */
 bool intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
                                            enum pipe pipe, bool enable);
+bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
+                                             enum pipe pipe, bool enable);
 bool intel_set_pch_fifo_underrun_reporting(struct drm_device *dev,
                                            enum transcoder pch_transcoder,
                                            bool enable);
@@ -591,8 +628,8 @@ void ilk_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask);
 void ilk_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask);
 void snb_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask);
 void snb_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask);
-void hsw_pc8_disable_interrupts(struct drm_device *dev);
-void hsw_pc8_restore_interrupts(struct drm_device *dev);
+void hsw_runtime_pm_disable_interrupts(struct drm_device *dev);
+void hsw_runtime_pm_restore_interrupts(struct drm_device *dev);
 
 
 /* intel_crt.c */
@@ -664,11 +701,10 @@ int intel_pin_and_fence_fb_obj(struct drm_device *dev,
                                struct drm_i915_gem_object *obj,
                                struct intel_ring_buffer *pipelined);
 void intel_unpin_fb_obj(struct drm_i915_gem_object *obj);
-int intel_framebuffer_init(struct drm_device *dev,
-                           struct intel_framebuffer *ifb,
+struct drm_framebuffer *
+__intel_framebuffer_create(struct drm_device *dev,
                            struct drm_mode_fb_cmd2 *mode_cmd,
                            struct drm_i915_gem_object *obj);
-void intel_framebuffer_fini(struct intel_framebuffer *fb);
 void intel_prepare_page_flip(struct drm_device *dev, int plane);
 void intel_finish_page_flip(struct drm_device *dev, int pipe);
 void intel_finish_page_flip_plane(struct drm_device *dev, int plane);
@@ -696,9 +732,8 @@ unsigned long intel_gen4_compute_page_offset(int *x, int *y,
                                              unsigned int bpp,
                                              unsigned int pitch);
 void intel_display_handle_reset(struct drm_device *dev);
-void hsw_enable_pc8_work(struct work_struct *__work);
-void hsw_enable_package_c8(struct drm_i915_private *dev_priv);
-void hsw_disable_package_c8(struct drm_i915_private *dev_priv);
+void hsw_enable_pc8(struct drm_i915_private *dev_priv);
+void hsw_disable_pc8(struct drm_i915_private *dev_priv);
 void intel_dp_get_m_n(struct intel_crtc *crtc,
                       struct intel_crtc_config *pipe_config);
 int intel_dotclock_calculate(int link_freq, const struct intel_link_m_n *m_n);
@@ -708,8 +743,13 @@ ironlake_check_encoder_dotclock(const struct intel_crtc_config *pipe_config,
 bool intel_crtc_active(struct drm_crtc *crtc);
 void hsw_enable_ips(struct intel_crtc *crtc);
 void hsw_disable_ips(struct intel_crtc *crtc);
-void intel_display_set_init_power(struct drm_device *dev, bool enable);
+void intel_display_set_init_power(struct drm_i915_private *dev, bool enable);
+enum intel_display_power_domain
+intel_display_port_power_domain(struct intel_encoder *intel_encoder);
 int valleyview_get_vco(struct drm_i915_private *dev_priv);
+void intel_mode_from_pipe_config(struct drm_display_mode *mode,
+                                 struct intel_crtc_config *pipe_config);
+int intel_format_to_fourcc(int format);
 
 /* intel_dp.c */
 void intel_dp_init(struct drm_device *dev, int output_reg, enum port port);
@@ -721,15 +761,15 @@ void intel_dp_stop_link_train(struct intel_dp *intel_dp);
 void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode);
 void intel_dp_encoder_destroy(struct drm_encoder *encoder);
 void intel_dp_check_link_status(struct intel_dp *intel_dp);
+int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc);
 bool intel_dp_compute_config(struct intel_encoder *encoder,
                              struct intel_crtc_config *pipe_config);
 bool intel_dp_is_edp(struct drm_device *dev, enum port port);
-void ironlake_edp_backlight_on(struct intel_dp *intel_dp);
-void ironlake_edp_backlight_off(struct intel_dp *intel_dp);
-void ironlake_edp_panel_on(struct intel_dp *intel_dp);
-void ironlake_edp_panel_off(struct intel_dp *intel_dp);
-void ironlake_edp_panel_vdd_on(struct intel_dp *intel_dp);
-void ironlake_edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync);
+void intel_edp_backlight_on(struct intel_dp *intel_dp);
+void intel_edp_backlight_off(struct intel_dp *intel_dp);
+void intel_edp_panel_vdd_on(struct intel_dp *intel_dp);
+void intel_edp_panel_on(struct intel_dp *intel_dp);
+void intel_edp_panel_off(struct intel_dp *intel_dp);
 void intel_edp_psr_enable(struct intel_dp *intel_dp);
 void intel_edp_psr_disable(struct intel_dp *intel_dp);
 void intel_edp_psr_update(struct drm_device *dev);
@@ -808,7 +848,8 @@ int intel_overlay_attrs(struct drm_device *dev, void *data,
 
 /* intel_panel.c */
 int intel_panel_init(struct intel_panel *panel,
-                     struct drm_display_mode *fixed_mode);
+                     struct drm_display_mode *fixed_mode,
+                     struct drm_display_mode *downclock_mode);
 void intel_panel_fini(struct intel_panel *panel);
 void intel_fixed_panel_mode(const struct drm_display_mode *fixed_mode,
                             struct drm_display_mode *adjusted_mode);
@@ -845,18 +886,19 @@ bool intel_fbc_enabled(struct drm_device *dev);
 void intel_update_fbc(struct drm_device *dev);
 void intel_gpu_ips_init(struct drm_i915_private *dev_priv);
 void intel_gpu_ips_teardown(void);
-int intel_power_domains_init(struct drm_device *dev);
-void intel_power_domains_remove(struct drm_device *dev);
-bool intel_display_power_enabled(struct drm_device *dev,
+int intel_power_domains_init(struct drm_i915_private *);
+void intel_power_domains_remove(struct drm_i915_private *);
+bool intel_display_power_enabled(struct drm_i915_private *dev_priv,
                                  enum intel_display_power_domain domain);
-bool intel_display_power_enabled_sw(struct drm_device *dev,
+bool intel_display_power_enabled_sw(struct drm_i915_private *dev_priv,
                                     enum intel_display_power_domain domain);
-void intel_display_power_get(struct drm_device *dev,
+void intel_display_power_get(struct drm_i915_private *dev_priv,
                              enum intel_display_power_domain domain);
-void intel_display_power_put(struct drm_device *dev,
+void intel_display_power_put(struct drm_i915_private *dev_priv,
                              enum intel_display_power_domain domain);
-void intel_power_domains_init_hw(struct drm_device *dev);
-void intel_set_power_well(struct drm_device *dev, bool enable);
+void intel_power_domains_init_hw(struct drm_i915_private *dev_priv);
+void intel_init_gt_powersave(struct drm_device *dev);
+void intel_cleanup_gt_powersave(struct drm_device *dev);
 void intel_enable_gt_powersave(struct drm_device *dev);
 void intel_disable_gt_powersave(struct drm_device *dev);
 void ironlake_teardown_rc6(struct drm_device *dev);
diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c
index fabbf0d895cf..33656647f8bc 100644
--- a/drivers/gpu/drm/i915/intel_dsi.c
+++ b/drivers/gpu/drm/i915/intel_dsi.c
@@ -243,11 +243,16 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
                                    enum pipe *pipe)
 {
         struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        enum intel_display_power_domain power_domain;
         u32 port, func;
         enum pipe p;
 
         DRM_DEBUG_KMS("\n");
 
+        power_domain = intel_display_port_power_domain(encoder);
+        if (!intel_display_power_enabled(dev_priv, power_domain))
+                return false;
+
         /* XXX: this only works for one DSI output */
         for (p = PIPE_A; p <= PIPE_B; p++) {
                 port = I915_READ(MIPI_PORT_CTRL(p));
@@ -488,8 +493,19 @@ static enum drm_connector_status
 intel_dsi_detect(struct drm_connector *connector, bool force)
 {
         struct intel_dsi *intel_dsi = intel_attached_dsi(connector);
+        struct intel_encoder *intel_encoder = &intel_dsi->base;
+        enum intel_display_power_domain power_domain;
+        enum drm_connector_status connector_status;
+        struct drm_i915_private *dev_priv = intel_encoder->base.dev->dev_private;
+
         DRM_DEBUG_KMS("\n");
-        return intel_dsi->dev.dev_ops->detect(&intel_dsi->dev);
+        power_domain = intel_display_port_power_domain(intel_encoder);
+
+        intel_display_power_get(dev_priv, power_domain);
+        connector_status = intel_dsi->dev.dev_ops->detect(&intel_dsi->dev);
+        intel_display_power_put(dev_priv, power_domain);
+
+        return connector_status;
 }
 
 static int intel_dsi_get_modes(struct drm_connector *connector)
@@ -586,6 +602,7 @@ bool intel_dsi_init(struct drm_device *dev)
         intel_encoder->get_config = intel_dsi_get_config;
 
         intel_connector->get_hw_state = intel_connector_get_hw_state;
+        intel_connector->unregister = intel_connector_unregister;
 
         for (i = 0; i < ARRAY_SIZE(intel_dsi_devices); i++) {
                 dsi = &intel_dsi_devices[i];
@@ -603,7 +620,7 @@ bool intel_dsi_init(struct drm_device *dev)
         intel_encoder->type = INTEL_OUTPUT_DSI;
         intel_encoder->crtc_mask = (1 << 0); /* XXX */
 
-        intel_encoder->cloneable = false;
+        intel_encoder->cloneable = 0;
         drm_connector_init(dev, connector, &intel_dsi_connector_funcs,
                            DRM_MODE_CONNECTOR_DSI);
 
@@ -624,7 +641,7 @@ bool intel_dsi_init(struct drm_device *dev)
         }
 
         fixed_mode->type |= DRM_MODE_TYPE_PREFERRED;
-        intel_panel_init(&intel_connector->panel, fixed_mode);
+        intel_panel_init(&intel_connector->panel, fixed_mode, NULL);
 
         return true;
 
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index eeff998e52ef..7fe3feedfe03 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -477,6 +477,7 @@ void intel_dvo_init(struct drm_device *dev)
         intel_encoder->compute_config = intel_dvo_compute_config;
         intel_encoder->mode_set = intel_dvo_mode_set;
         intel_connector->get_hw_state = intel_dvo_connector_get_hw_state;
+        intel_connector->unregister = intel_connector_unregister;
 
         /* Now, try to find a controller */
         for (i = 0; i < ARRAY_SIZE(intel_dvo_devices); i++) {
@@ -521,14 +522,15 @@ void intel_dvo_init(struct drm_device *dev)
                 intel_encoder->crtc_mask = (1 << 0) | (1 << 1);
                 switch (dvo->type) {
                 case INTEL_DVO_CHIP_TMDS:
-                        intel_encoder->cloneable = true;
+                        intel_encoder->cloneable = (1 << INTEL_OUTPUT_ANALOG) |
+                                (1 << INTEL_OUTPUT_DVO);
                         drm_connector_init(dev, connector,
                                            &intel_dvo_connector_funcs,
                                            DRM_MODE_CONNECTOR_DVII);
                         encoder_type = DRM_MODE_ENCODER_TMDS;
                         break;
                 case INTEL_DVO_CHIP_LVDS:
-                        intel_encoder->cloneable = false;
+                        intel_encoder->cloneable = 0;
                         drm_connector_init(dev, connector,
                                            &intel_dvo_connector_funcs,
                                            DRM_MODE_CONNECTOR_LVDS);
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index 39eac9937a4a..b4d44e62f0c7 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -62,6 +62,7 @@ static int intelfb_alloc(struct drm_fb_helper *helper,
 {
         struct intel_fbdev *ifbdev =
                 container_of(helper, struct intel_fbdev, helper);
+        struct drm_framebuffer *fb;
         struct drm_device *dev = helper->dev;
         struct drm_mode_fb_cmd2 mode_cmd = {};
         struct drm_i915_gem_object *obj;
@@ -93,18 +94,22 @@ static int intelfb_alloc(struct drm_fb_helper *helper,
         /* Flush everything out, we'll be doing GTT only from now on */
         ret = intel_pin_and_fence_fb_obj(dev, obj, NULL);
         if (ret) {
-                DRM_ERROR("failed to pin fb: %d\n", ret);
+                DRM_ERROR("failed to pin obj: %d\n", ret);
                 goto out_unref;
         }
 
-        ret = intel_framebuffer_init(dev, &ifbdev->ifb, &mode_cmd, obj);
-        if (ret)
+        fb = __intel_framebuffer_create(dev, &mode_cmd, obj);
+        if (IS_ERR(fb)) {
+                ret = PTR_ERR(fb);
                 goto out_unpin;
+        }
+
+        ifbdev->fb = to_intel_framebuffer(fb);
 
         return 0;
 
 out_unpin:
-        i915_gem_object_unpin(obj);
+        i915_gem_object_ggtt_unpin(obj);
 out_unref:
         drm_gem_object_unreference(&obj->base);
 out:
@@ -116,23 +121,26 @@ static int intelfb_create(struct drm_fb_helper *helper,
 {
         struct intel_fbdev *ifbdev =
                 container_of(helper, struct intel_fbdev, helper);
-        struct intel_framebuffer *intel_fb = &ifbdev->ifb;
+        struct intel_framebuffer *intel_fb = ifbdev->fb;
         struct drm_device *dev = helper->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct fb_info *info;
         struct drm_framebuffer *fb;
         struct drm_i915_gem_object *obj;
         int size, ret;
+        bool prealloc = false;
 
         mutex_lock(&dev->struct_mutex);
 
-        if (!intel_fb->obj) {
+        if (!intel_fb || WARN_ON(!intel_fb->obj)) {
                 DRM_DEBUG_KMS("no BIOS fb, allocating a new one\n");
                 ret = intelfb_alloc(helper, sizes);
                 if (ret)
                         goto out_unlock;
+                intel_fb = ifbdev->fb;
         } else {
                 DRM_DEBUG_KMS("re-using BIOS fb\n");
+                prealloc = true;
                 sizes->fb_width = intel_fb->base.width;
                 sizes->fb_height = intel_fb->base.height;
         }
@@ -148,7 +156,7 @@ static int intelfb_create(struct drm_fb_helper *helper,
 
         info->par = helper;
 
-        fb = &ifbdev->ifb.base;
+        fb = &ifbdev->fb->base;
 
         ifbdev->helper.fb = fb;
         ifbdev->helper.fbdev = info;
@@ -194,7 +202,7 @@ static int intelfb_create(struct drm_fb_helper *helper,
          * If the object is stolen however, it will be full of whatever
          * garbage was left in there.
          */
-        if (ifbdev->ifb.obj->stolen)
+        if (ifbdev->fb->obj->stolen && !prealloc)
                 memset_io(info->screen_base, 0, info->screen_size);
 
         /* Use default scratch pixmap (info->pixmap.flags = FB_PIXMAP_SYSTEM) */
@@ -208,7 +216,7 @@ static int intelfb_create(struct drm_fb_helper *helper,
         return 0;
 
 out_unpin:
-        i915_gem_object_unpin(obj);
+        i915_gem_object_ggtt_unpin(obj);
         drm_gem_object_unreference(&obj->base);
 out_unlock:
         mutex_unlock(&dev->struct_mutex);
@@ -236,7 +244,193 @@ static void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
         *blue = intel_crtc->lut_b[regno] << 8;
 }
 
+static struct drm_fb_helper_crtc *
+intel_fb_helper_crtc(struct drm_fb_helper *fb_helper, struct drm_crtc *crtc)
+{
+        int i;
+
+        for (i = 0; i < fb_helper->crtc_count; i++)
+                if (fb_helper->crtc_info[i].mode_set.crtc == crtc)
+                        return &fb_helper->crtc_info[i];
+
+        return NULL;
+}
+
+/*
+ * Try to read the BIOS display configuration and use it for the initial
+ * fb configuration.
+ *
+ * The BIOS or boot loader will generally create an initial display
+ * configuration for us that includes some set of active pipes and displays.
+ * This routine tries to figure out which pipes and connectors are active
+ * and stuffs them into the crtcs and modes array given to us by the
+ * drm_fb_helper code.
+ *
+ * The overall sequence is:
+ *   intel_fbdev_init - from driver load
+ *     intel_fbdev_init_bios - initialize the intel_fbdev using BIOS data
+ *     drm_fb_helper_init - build fb helper structs
+ *     drm_fb_helper_single_add_all_connectors - more fb helper structs
+ *   intel_fbdev_initial_config - apply the config
+ *     drm_fb_helper_initial_config - call ->probe then register_framebuffer()
+ *         drm_setup_crtcs - build crtc config for fbdev
+ *           intel_fb_initial_config - find active connectors etc
+ *         drm_fb_helper_single_fb_probe - set up fbdev
+ *           intelfb_create - re-use or alloc fb, build out fbdev structs
+ *
+ * Note that we don't make special consideration whether we could actually
+ * switch to the selected modes without a full modeset. E.g. when the display
+ * is in VGA mode we need to recalculate watermarks and set a new high-res
+ * framebuffer anyway.
+ */
+static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
+                                    struct drm_fb_helper_crtc **crtcs,
+                                    struct drm_display_mode **modes,
+                                    bool *enabled, int width, int height)
+{
+        struct drm_device *dev = fb_helper->dev;
+        int i, j;
+        bool *save_enabled;
+        bool fallback = true;
+        int num_connectors_enabled = 0;
+        int num_connectors_detected = 0;
+
+        /*
+         * If the user specified any force options, just bail here
+         * and use that config.
+         */
+        for (i = 0; i < fb_helper->connector_count; i++) {
+                struct drm_fb_helper_connector *fb_conn;
+                struct drm_connector *connector;
+
+                fb_conn = fb_helper->connector_info[i];
+                connector = fb_conn->connector;
+
+                if (!enabled[i])
+                        continue;
+
+                if (connector->force != DRM_FORCE_UNSPECIFIED)
+                        return false;
+        }
+
+        save_enabled = kcalloc(dev->mode_config.num_connector, sizeof(bool),
+                               GFP_KERNEL);
+        if (!save_enabled)
+                return false;
+
+        memcpy(save_enabled, enabled, dev->mode_config.num_connector);
+
+        for (i = 0; i < fb_helper->connector_count; i++) {
+                struct drm_fb_helper_connector *fb_conn;
+                struct drm_connector *connector;
+                struct drm_encoder *encoder;
+                struct drm_fb_helper_crtc *new_crtc;
+
+                fb_conn = fb_helper->connector_info[i];
+                connector = fb_conn->connector;
+
+                if (connector->status == connector_status_connected)
+                        num_connectors_detected++;
+
+                if (!enabled[i]) {
+                        DRM_DEBUG_KMS("connector %d not enabled, skipping\n",
+                                      connector->base.id);
+                        continue;
+                }
+
+                encoder = connector->encoder;
+                if (!encoder || WARN_ON(!encoder->crtc)) {
+                        DRM_DEBUG_KMS("connector %d has no encoder or crtc, skipping\n",
+                                      connector->base.id);
+                        enabled[i] = false;
+                        continue;
+                }
+
+                num_connectors_enabled++;
+
+                new_crtc = intel_fb_helper_crtc(fb_helper, encoder->crtc);
+
+                /*
+                 * Make sure we're not trying to drive multiple connectors
+                 * with a single CRTC, since our cloning support may not
+                 * match the BIOS.
+                 */
+                for (j = 0; j < fb_helper->connector_count; j++) {
+                        if (crtcs[j] == new_crtc) {
+                                DRM_DEBUG_KMS("fallback: cloned configuration\n");
+                                fallback = true;
+                                goto out;
+                        }
+                }
+
+                DRM_DEBUG_KMS("looking for cmdline mode on connector %d\n",
+                              fb_conn->connector->base.id);
+
+                /* go for command line mode first */
+                modes[i] = drm_pick_cmdline_mode(fb_conn, width, height);
+
+                /* try for preferred next */
+                if (!modes[i]) {
+                        DRM_DEBUG_KMS("looking for preferred mode on connector %d\n",
+                                      fb_conn->connector->base.id);
+                        modes[i] = drm_has_preferred_mode(fb_conn, width,
+                                                          height);
+                }
+
+                /* last resort: use current mode */
+                if (!modes[i]) {
+                        /*
+                         * IMPORTANT: We want to use the adjusted mode (i.e.
+                         * after the panel fitter upscaling) as the initial
+                         * config, not the input mode, which is what crtc->mode
+                         * usually contains. But since our current fastboot
+                         * code puts a mode derived from the post-pfit timings
+                         * into crtc->mode this works out correctly. We don't
+                         * use hwmode anywhere right now, so use it for this
+                         * since the fb helper layer wants a pointer to
+                         * something we own.
+                         */
+                        intel_mode_from_pipe_config(&encoder->crtc->hwmode,
+                                                    &to_intel_crtc(encoder->crtc)->config);
+                        modes[i] = &encoder->crtc->hwmode;
+                }
+                crtcs[i] = new_crtc;
+
+                DRM_DEBUG_KMS("connector %s on crtc %d: %s\n",
+                              drm_get_connector_name(connector),
+                              encoder->crtc->base.id,
+                              modes[i]->name);
+
+                fallback = false;
+        }
+
+        /*
+         * If the BIOS didn't enable everything it could, fall back to have the
+         * same user experiencing of lighting up as much as possible like the
+         * fbdev helper library.
+         */
+        if (num_connectors_enabled != num_connectors_detected &&
+            num_connectors_enabled < INTEL_INFO(dev)->num_pipes) {
+                DRM_DEBUG_KMS("fallback: Not all outputs enabled\n");
+                DRM_DEBUG_KMS("Enabled: %i, detected: %i\n", num_connectors_enabled,
+                              num_connectors_detected);
+                fallback = true;
+        }
+
+out:
+        if (fallback) {
+                DRM_DEBUG_KMS("Not using firmware configuration\n");
+                memcpy(enabled, save_enabled, dev->mode_config.num_connector);
+                kfree(save_enabled);
+                return false;
+        }
+
+        kfree(save_enabled);
+        return true;
+}
+
 static struct drm_fb_helper_funcs intel_fb_helper_funcs = {
+        .initial_config = intel_fb_initial_config,
         .gamma_set = intel_crtc_fb_gamma_set,
         .gamma_get = intel_crtc_fb_gamma_get,
         .fb_probe = intelfb_create,
@@ -258,8 +452,139 @@ static void intel_fbdev_destroy(struct drm_device *dev,
 
         drm_fb_helper_fini(&ifbdev->helper);
 
-        drm_framebuffer_unregister_private(&ifbdev->ifb.base);
-        intel_framebuffer_fini(&ifbdev->ifb);
+        drm_framebuffer_unregister_private(&ifbdev->fb->base);
+        drm_framebuffer_remove(&ifbdev->fb->base);
+}
+
+/*
+ * Build an intel_fbdev struct using a BIOS allocated framebuffer, if possible.
+ * The core display code will have read out the current plane configuration,
+ * so we use that to figure out if there's an object for us to use as the
+ * fb, and if so, we re-use it for the fbdev configuration.
+ *
+ * Note we only support a single fb shared across pipes for boot (mostly for
+ * fbcon), so we just find the biggest and use that.
+ */
+static bool intel_fbdev_init_bios(struct drm_device *dev,
+                                 struct intel_fbdev *ifbdev)
+{
+        struct intel_framebuffer *fb = NULL;
+        struct drm_crtc *crtc;
+        struct intel_crtc *intel_crtc;
+        struct intel_plane_config *plane_config = NULL;
+        unsigned int max_size = 0;
+
+        if (!i915.fastboot)
+                return false;
+
+        /* Find the largest fb */
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                intel_crtc = to_intel_crtc(crtc);
+
+                if (!intel_crtc->active || !crtc->primary->fb) {
+                        DRM_DEBUG_KMS("pipe %c not active or no fb, skipping\n",
+                                      pipe_name(intel_crtc->pipe));
+                        continue;
+                }
+
+                if (intel_crtc->plane_config.size > max_size) {
+                        DRM_DEBUG_KMS("found possible fb from plane %c\n",
+                                      pipe_name(intel_crtc->pipe));
+                        plane_config = &intel_crtc->plane_config;
+                        fb = to_intel_framebuffer(crtc->primary->fb);
+                        max_size = plane_config->size;
+                }
+        }
+
+        if (!fb) {
+                DRM_DEBUG_KMS("no active fbs found, not using BIOS config\n");
+                goto out;
+        }
+
+        /* Now make sure all the pipes will fit into it */
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                unsigned int cur_size;
+
+                intel_crtc = to_intel_crtc(crtc);
+
+                if (!intel_crtc->active) {
+                        DRM_DEBUG_KMS("pipe %c not active, skipping\n",
+                                      pipe_name(intel_crtc->pipe));
+                        continue;
+                }
+
+                DRM_DEBUG_KMS("checking plane %c for BIOS fb\n",
+                              pipe_name(intel_crtc->pipe));
+
+                /*
+                 * See if the plane fb we found above will fit on this
+                 * pipe.  Note we need to use the selected fb's pitch and bpp
+                 * rather than the current pipe's, since they differ.
+                 */
+                cur_size = intel_crtc->config.adjusted_mode.crtc_hdisplay;
+                cur_size = cur_size * fb->base.bits_per_pixel / 8;
+                if (fb->base.pitches[0] < cur_size) {
+                        DRM_DEBUG_KMS("fb not wide enough for plane %c (%d vs %d)\n",
+                                      pipe_name(intel_crtc->pipe),
+                                      cur_size, fb->base.pitches[0]);
+                        plane_config = NULL;
+                        fb = NULL;
+                        break;
+                }
+
+                cur_size = intel_crtc->config.adjusted_mode.crtc_vdisplay;
+                cur_size = ALIGN(cur_size, plane_config->tiled ? (IS_GEN2(dev) ? 16 : 8) : 1);
+                cur_size *= fb->base.pitches[0];
+                DRM_DEBUG_KMS("pipe %c area: %dx%d, bpp: %d, size: %d\n",
+                              pipe_name(intel_crtc->pipe),
+                              intel_crtc->config.adjusted_mode.crtc_hdisplay,
+                              intel_crtc->config.adjusted_mode.crtc_vdisplay,
+                              fb->base.bits_per_pixel,
+                              cur_size);
+
+                if (cur_size > max_size) {
+                        DRM_DEBUG_KMS("fb not big enough for plane %c (%d vs %d)\n",
+                                      pipe_name(intel_crtc->pipe),
+                                      cur_size, max_size);
+                        plane_config = NULL;
+                        fb = NULL;
+                        break;
+                }
+
+                DRM_DEBUG_KMS("fb big enough for plane %c (%d >= %d)\n",
+                              pipe_name(intel_crtc->pipe),
+                              max_size, cur_size);
+        }
+
+        if (!fb) {
+                DRM_DEBUG_KMS("BIOS fb not suitable for all pipes, not using\n");
+                goto out;
+        }
+
+        ifbdev->preferred_bpp = fb->base.bits_per_pixel;
+        ifbdev->fb = fb;
+
+        drm_framebuffer_reference(&ifbdev->fb->base);
+
+        /* Final pass to check if any active pipes don't have fbs */
+        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+                intel_crtc = to_intel_crtc(crtc);
+
+                if (!intel_crtc->active)
+                        continue;
+
+                WARN(!crtc->primary->fb,
+                     "re-used BIOS config but lost an fb on crtc %d\n",
+                     crtc->base.id);
+        }
+
+
+        DRM_DEBUG_KMS("using BIOS fb for initial console\n");
+        return true;
+
+out:
+
+        return false;
 }
 
 int intel_fbdev_init(struct drm_device *dev)
@@ -268,21 +593,25 @@ int intel_fbdev_init(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
-        ifbdev = kzalloc(sizeof(*ifbdev), GFP_KERNEL);
-        if (!ifbdev)
+        if (WARN_ON(INTEL_INFO(dev)->num_pipes == 0))
+                return -ENODEV;
+
+        ifbdev = kzalloc(sizeof(struct intel_fbdev), GFP_KERNEL);
+        if (ifbdev == NULL)
                 return -ENOMEM;
 
-        dev_priv->fbdev = ifbdev;
         ifbdev->helper.funcs = &intel_fb_helper_funcs;
+        if (!intel_fbdev_init_bios(dev, ifbdev))
+                ifbdev->preferred_bpp = 32;
 
         ret = drm_fb_helper_init(dev, &ifbdev->helper,
-                                 INTEL_INFO(dev)->num_pipes,
-                                 4);
+                                 INTEL_INFO(dev)->num_pipes, 4);
         if (ret) {
                 kfree(ifbdev);
                 return ret;
         }
 
+        dev_priv->fbdev = ifbdev;
         drm_fb_helper_single_add_all_connectors(&ifbdev->helper);
 
         return 0;
@@ -291,9 +620,10 @@ int intel_fbdev_init(struct drm_device *dev)
 void intel_fbdev_initial_config(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_fbdev *ifbdev = dev_priv->fbdev;
 
         /* Due to peculiar init order wrt to hpd handling this is separate. */
-        drm_fb_helper_initial_config(&dev_priv->fbdev->helper, 32);
+        drm_fb_helper_initial_config(&ifbdev->helper, ifbdev->preferred_bpp);
 }
 
 void intel_fbdev_fini(struct drm_device *dev)
@@ -322,7 +652,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state)
          * been restored from swap. If the object is stolen however, it will be
          * full of whatever garbage was left in there.
          */
-        if (state == FBINFO_STATE_RUNNING && ifbdev->ifb.obj->stolen)
+        if (state == FBINFO_STATE_RUNNING && ifbdev->fb->obj->stolen)
                 memset_io(info->screen_base, 0, info->screen_size);
 
         fb_set_suspend(info, state);
@@ -331,7 +661,8 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state)
 void intel_fbdev_output_poll_changed(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        drm_fb_helper_hotplug_event(&dev_priv->fbdev->helper);
+        if (dev_priv->fbdev)
+                drm_fb_helper_hotplug_event(&dev_priv->fbdev->helper);
 }
 
 void intel_fbdev_restore_mode(struct drm_device *dev)
@@ -339,7 +670,7 @@ void intel_fbdev_restore_mode(struct drm_device *dev)
         int ret;
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        if (INTEL_INFO(dev)->num_pipes == 0)
+        if (!dev_priv->fbdev)
                 return;
 
         drm_modeset_lock_all(dev);
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index ee3181ebcc92..b0413e190625 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -113,7 +113,8 @@ static u32 hsw_infoframe_enable(enum hdmi_infoframe_type type)
 }
 
 static u32 hsw_infoframe_data_reg(enum hdmi_infoframe_type type,
-                                  enum transcoder cpu_transcoder)
+                                  enum transcoder cpu_transcoder,
+                                  struct drm_i915_private *dev_priv)
 {
         switch (type) {
         case HDMI_INFOFRAME_TYPE_AVI:
@@ -296,7 +297,8 @@ static void hsw_write_infoframe(struct drm_encoder *encoder,
         u32 val = I915_READ(ctl_reg);
 
         data_reg = hsw_infoframe_data_reg(type,
-                                          intel_crtc->config.cpu_transcoder);
+                                          intel_crtc->config.cpu_transcoder,
+                                          dev_priv);
         if (data_reg == 0)
                 return;
 
@@ -423,7 +425,7 @@ static void g4x_set_infoframes(struct drm_encoder *encoder,
         struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi;
         u32 reg = VIDEO_DIP_CTL;
         u32 val = I915_READ(reg);
-        u32 port;
+        u32 port = VIDEO_DIP_PORT(intel_dig_port->port);
 
         assert_hdmi_port_disabled(intel_hdmi);
 
@@ -447,18 +449,6 @@ static void g4x_set_infoframes(struct drm_encoder *encoder,
                 return;
         }
 
-        switch (intel_dig_port->port) {
-        case PORT_B:
-                port = VIDEO_DIP_PORT_B;
-                break;
-        case PORT_C:
-                port = VIDEO_DIP_PORT_C;
-                break;
-        default:
-                BUG();
-                return;
-        }
-
         if (port != (val & VIDEO_DIP_PORT_MASK)) {
                 if (val & VIDEO_DIP_ENABLE) {
                         val &= ~VIDEO_DIP_ENABLE;
@@ -489,7 +479,7 @@ static void ibx_set_infoframes(struct drm_encoder *encoder,
         struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi;
         u32 reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
         u32 val = I915_READ(reg);
-        u32 port;
+        u32 port = VIDEO_DIP_PORT(intel_dig_port->port);
 
         assert_hdmi_port_disabled(intel_hdmi);
 
@@ -505,21 +495,6 @@ static void ibx_set_infoframes(struct drm_encoder *encoder,
                 return;
         }
 
-        switch (intel_dig_port->port) {
-        case PORT_B:
-                port = VIDEO_DIP_PORT_B;
-                break;
-        case PORT_C:
-                port = VIDEO_DIP_PORT_C;
-                break;
-        case PORT_D:
-                port = VIDEO_DIP_PORT_D;
-                break;
-        default:
-                BUG();
-                return;
-        }
-
         if (port != (val & VIDEO_DIP_PORT_MASK)) {
                 if (val & VIDEO_DIP_ENABLE) {
                         val &= ~VIDEO_DIP_ENABLE;
@@ -692,8 +667,13 @@ static bool intel_hdmi_get_hw_state(struct intel_encoder *encoder,
         struct drm_device *dev = encoder->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
+        enum intel_display_power_domain power_domain;
         u32 tmp;
 
+        power_domain = intel_display_port_power_domain(encoder);
+        if (!intel_display_power_enabled(dev_priv, power_domain))
+                return false;
+
         tmp = I915_READ(intel_hdmi->hdmi_reg);
 
         if (!(tmp & SDVO_ENABLE))
@@ -868,6 +848,30 @@ intel_hdmi_mode_valid(struct drm_connector *connector,
         return MODE_OK;
 }
 
+static bool hdmi_12bpc_possible(struct intel_crtc *crtc)
+{
+        struct drm_device *dev = crtc->base.dev;
+        struct intel_encoder *encoder;
+        int count = 0, count_hdmi = 0;
+
+        if (!HAS_PCH_SPLIT(dev))
+                return false;
+
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
+                if (encoder->new_crtc != crtc)
+                        continue;
+
+                count_hdmi += encoder->type == INTEL_OUTPUT_HDMI;
+                count++;
+        }
+
+        /*
+         * HDMI 12bpc affects the clocks, so it's only possible
+         * when not cloning with other encoder types.
+         */
+        return count_hdmi > 0 && count_hdmi == count;
+}
+
 bool intel_hdmi_compute_config(struct intel_encoder *encoder,
                                struct intel_crtc_config *pipe_config)
 {
@@ -900,7 +904,8 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
          * within limits.
          */
         if (pipe_config->pipe_bpp > 8*3 && intel_hdmi->has_hdmi_sink &&
-            clock_12bpc <= portclock_limit && HAS_PCH_SPLIT(dev)) {
+            clock_12bpc <= portclock_limit &&
+            hdmi_12bpc_possible(encoder->new_crtc)) {
                 DRM_DEBUG_KMS("picking bpc to 12 for HDMI output\n");
                 desired_bpp = 12*3;
 
@@ -934,11 +939,15 @@ intel_hdmi_detect(struct drm_connector *connector, bool force)
         struct intel_encoder *intel_encoder = &intel_dig_port->base;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct edid *edid;
+        enum intel_display_power_domain power_domain;
         enum drm_connector_status status = connector_status_disconnected;
 
         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
                       connector->base.id, drm_get_connector_name(connector));
 
+        power_domain = intel_display_port_power_domain(intel_encoder);
+        intel_display_power_get(dev_priv, power_domain);
+
         intel_hdmi->has_hdmi_sink = false;
         intel_hdmi->has_audio = false;
         intel_hdmi->rgb_quant_range_selectable = false;
@@ -966,31 +975,48 @@ intel_hdmi_detect(struct drm_connector *connector, bool force)
                 intel_encoder->type = INTEL_OUTPUT_HDMI;
         }
 
+        intel_display_power_put(dev_priv, power_domain);
+
         return status;
 }
 
 static int intel_hdmi_get_modes(struct drm_connector *connector)
 {
-        struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
+        struct intel_encoder *intel_encoder = intel_attached_encoder(connector);
+        struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base);
         struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        enum intel_display_power_domain power_domain;
+        int ret;
 
         /* We should parse the EDID data and find out if it's an HDMI sink so
          * we can send audio to it.
          */
 
-        return intel_ddc_get_modes(connector,
+        power_domain = intel_display_port_power_domain(intel_encoder);
+        intel_display_power_get(dev_priv, power_domain);
+
+        ret = intel_ddc_get_modes(connector,
                                    intel_gmbus_get_adapter(dev_priv,
                                                            intel_hdmi->ddc_bus));
+
+        intel_display_power_put(dev_priv, power_domain);
+
+        return ret;
 }
 
 static bool
 intel_hdmi_detect_audio(struct drm_connector *connector)
 {
-        struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
+        struct intel_encoder *intel_encoder = intel_attached_encoder(connector);
+        struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base);
         struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        enum intel_display_power_domain power_domain;
         struct edid *edid;
         bool has_audio = false;
 
+        power_domain = intel_display_port_power_domain(intel_encoder);
+        intel_display_power_get(dev_priv, power_domain);
+
         edid = drm_get_edid(connector,
                             intel_gmbus_get_adapter(dev_priv,
                                                     intel_hdmi->ddc_bus));
@@ -1000,6 +1026,8 @@ intel_hdmi_detect_audio(struct drm_connector *connector)
                 kfree(edid);
         }
 
+        intel_display_power_put(dev_priv, power_domain);
+
         return has_audio;
 }
 
@@ -1261,6 +1289,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
                 intel_connector->get_hw_state = intel_ddi_connector_get_hw_state;
         else
                 intel_connector->get_hw_state = intel_connector_get_hw_state;
+        intel_connector->unregister = intel_connector_unregister;
 
         intel_hdmi_add_properties(intel_hdmi, connector);
 
@@ -1314,7 +1343,14 @@ void intel_hdmi_init(struct drm_device *dev, int hdmi_reg, enum port port)
 
         intel_encoder->type = INTEL_OUTPUT_HDMI;
         intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
-        intel_encoder->cloneable = false;
+        intel_encoder->cloneable = 1 << INTEL_OUTPUT_ANALOG;
+        /*
+         * BSpec is unclear about HDMI+HDMI cloning on g4x, but it seems
+         * to work on real hardware. And since g4x can send infoframes to
+         * only one port anyway, nothing is lost by allowing it.
+         */
+        if (IS_G4X(dev))
+                intel_encoder->cloneable |= 1 << INTEL_OUTPUT_HDMI;
 
         intel_dig_port->port = port;
         intel_dig_port->hdmi.hdmi_reg = hdmi_reg;
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index 8bcb93a2a9f6..f1ecf916474a 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -848,8 +848,8 @@ static bool compute_is_dual_link_lvds(struct intel_lvds_encoder *lvds_encoder)
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         /* use the module option value if specified */
-        if (i915_lvds_channel_mode > 0)
-                return i915_lvds_channel_mode == 2;
+        if (i915.lvds_channel_mode > 0)
+                return i915.lvds_channel_mode == 2;
 
         if (dmi_check_system(intel_dual_link_lvds))
                 return true;
@@ -899,6 +899,7 @@ void intel_lvds_init(struct drm_device *dev)
         struct drm_encoder *encoder;
         struct drm_display_mode *scan; /* *modes, *bios_mode; */
         struct drm_display_mode *fixed_mode = NULL;
+        struct drm_display_mode *downclock_mode = NULL;
         struct edid *edid;
         struct drm_crtc *crtc;
         u32 lvds;
@@ -957,11 +958,12 @@ void intel_lvds_init(struct drm_device *dev)
         intel_encoder->get_hw_state = intel_lvds_get_hw_state;
         intel_encoder->get_config = intel_lvds_get_config;
         intel_connector->get_hw_state = intel_connector_get_hw_state;
+        intel_connector->unregister = intel_connector_unregister;
 
         intel_connector_attach_encoder(intel_connector, intel_encoder);
         intel_encoder->type = INTEL_OUTPUT_LVDS;
 
-        intel_encoder->cloneable = false;
+        intel_encoder->cloneable = 0;
         if (HAS_PCH_SPLIT(dev))
                 intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
         else if (IS_GEN4(dev))
@@ -1000,6 +1002,7 @@ void intel_lvds_init(struct drm_device *dev)
          * Attempt to get the fixed panel mode from DDC.  Assume that the
          * preferred mode is the right one.
          */
+        mutex_lock(&dev->mode_config.mutex);
         edid = drm_get_edid(connector, intel_gmbus_get_adapter(dev_priv, pin));
         if (edid) {
                 if (drm_add_edid_modes(connector, edid)) {
@@ -1032,15 +1035,14 @@ void intel_lvds_init(struct drm_device *dev)
 
                         fixed_mode = drm_mode_duplicate(dev, scan);
                         if (fixed_mode) {
-                                intel_connector->panel.downclock_mode =
+                                downclock_mode =
                                         intel_find_panel_downclock(dev,
                                         fixed_mode, connector);
-                                if (intel_connector->panel.downclock_mode !=
-                                        NULL && i915_lvds_downclock) {
+                                if (downclock_mode != NULL &&
+                                        i915.lvds_downclock) {
                                         /* We found the downclock for LVDS. */
                                         dev_priv->lvds_downclock_avail = true;
                                         dev_priv->lvds_downclock =
-                                                intel_connector->panel.
                                                 downclock_mode->clock;
                                         DRM_DEBUG_KMS("LVDS downclock is found"
                                         " in EDID. Normal clock %dKhz, "
@@ -1094,6 +1096,8 @@ void intel_lvds_init(struct drm_device *dev)
                 goto failed;
 
 out:
+        mutex_unlock(&dev->mode_config.mutex);
+
         lvds_encoder->is_dual_link = compute_is_dual_link_lvds(lvds_encoder);
         DRM_DEBUG_KMS("detected %s-link lvds configuration\n",
                       lvds_encoder->is_dual_link ? "dual" : "single");
@@ -1116,17 +1120,17 @@ out:
         }
         drm_sysfs_connector_add(connector);
 
-        intel_panel_init(&intel_connector->panel, fixed_mode);
+        intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode);
         intel_panel_setup_backlight(connector);
 
         return;
 
 failed:
+        mutex_unlock(&dev->mode_config.mutex);
+
         DRM_DEBUG_KMS("No LVDS modes found, disabling.\n");
         drm_connector_cleanup(connector);
         drm_encoder_cleanup(encoder);
-        if (fixed_mode)
-                drm_mode_destroy(dev, fixed_mode);
         kfree(lvds_encoder);
         kfree(lvds_connector);
         return;
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index a759ecdb7a6e..d8adc9104dca 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -189,7 +189,7 @@ struct intel_overlay {
 static struct overlay_registers __iomem *
 intel_overlay_map_regs(struct intel_overlay *overlay)
 {
-        drm_i915_private_t *dev_priv = overlay->dev->dev_private;
+        struct drm_i915_private *dev_priv = overlay->dev->dev_private;
         struct overlay_registers __iomem *regs;
 
         if (OVERLAY_NEEDS_PHYSICAL(overlay->dev))
@@ -212,7 +212,7 @@ static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
                                          void (*tail)(struct intel_overlay *))
 {
         struct drm_device *dev = overlay->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
         int ret;
 
@@ -262,7 +262,7 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
                                   bool load_polyphase_filter)
 {
         struct drm_device *dev = overlay->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
         u32 flip_addr = overlay->flip_addr;
         u32 tmp;
@@ -293,7 +293,7 @@ static void intel_overlay_release_old_vid_tail(struct intel_overlay *overlay)
 {
         struct drm_i915_gem_object *obj = overlay->old_vid_bo;
 
-        i915_gem_object_unpin(obj);
+        i915_gem_object_ggtt_unpin(obj);
         drm_gem_object_unreference(&obj->base);
 
         overlay->old_vid_bo = NULL;
@@ -306,7 +306,7 @@ static void intel_overlay_off_tail(struct intel_overlay *overlay)
         /* never have the overlay hw on without showing a frame */
         BUG_ON(!overlay->vid_bo);
 
-        i915_gem_object_unpin(obj);
+        i915_gem_object_ggtt_unpin(obj);
         drm_gem_object_unreference(&obj->base);
         overlay->vid_bo = NULL;
 
@@ -362,7 +362,7 @@ static int intel_overlay_off(struct intel_overlay *overlay)
 static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay)
 {
         struct drm_device *dev = overlay->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
         int ret;
 
@@ -388,7 +388,7 @@ static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay)
 static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
 {
         struct drm_device *dev = overlay->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
         int ret;
 
@@ -606,14 +606,14 @@ static void update_colorkey(struct intel_overlay *overlay,
 {
         u32 key = overlay->color_key;
 
-        switch (overlay->crtc->base.fb->bits_per_pixel) {
+        switch (overlay->crtc->base.primary->fb->bits_per_pixel) {
         case 8:
                 iowrite32(0, &regs->DCLRKV);
                 iowrite32(CLK_RGB8I_MASK | DST_KEY_ENABLE, &regs->DCLRKM);
                 break;
 
         case 16:
-                if (overlay->crtc->base.fb->depth == 15) {
+                if (overlay->crtc->base.primary->fb->depth == 15) {
                         iowrite32(RGB15_TO_COLORKEY(key), &regs->DCLRKV);
                         iowrite32(CLK_RGB15_MASK | DST_KEY_ENABLE,
                                   &regs->DCLRKM);
@@ -782,7 +782,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
         return 0;
 
 out_unpin:
-        i915_gem_object_unpin(new_bo);
+        i915_gem_object_ggtt_unpin(new_bo);
         return ret;
 }
 
@@ -834,7 +834,7 @@ static int check_overlay_possible_on_crtc(struct intel_overlay *overlay,
 static void update_pfit_vscale_ratio(struct intel_overlay *overlay)
 {
         struct drm_device *dev = overlay->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 pfit_control = I915_READ(PFIT_CONTROL);
         u32 ratio;
 
@@ -1026,7 +1026,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
                             struct drm_file *file_priv)
 {
         struct drm_intel_overlay_put_image *put_image_rec = data;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_overlay *overlay;
         struct drm_mode_object *drmmode_obj;
         struct intel_crtc *crtc;
@@ -1076,7 +1076,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
         mutex_lock(&dev->struct_mutex);
 
         if (new_bo->tiling_mode) {
-                DRM_ERROR("buffer used for overlay image can not be tiled\n");
+                DRM_DEBUG_KMS("buffer used for overlay image can not be tiled\n");
                 ret = -EINVAL;
                 goto out_unlock;
         }
@@ -1226,7 +1226,7 @@ int intel_overlay_attrs(struct drm_device *dev, void *data,
                         struct drm_file *file_priv)
 {
         struct drm_intel_overlay_attrs *attrs = data;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_overlay *overlay;
         struct overlay_registers __iomem *regs;
         int ret;
@@ -1311,7 +1311,7 @@ out_unlock:
 
 void intel_setup_overlay(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_overlay *overlay;
         struct drm_i915_gem_object *reg_bo;
         struct overlay_registers __iomem *regs;
@@ -1349,7 +1349,7 @@ void intel_setup_overlay(struct drm_device *dev)
                 }
                 overlay->flip_addr = reg_bo->phys_obj->handle->busaddr;
         } else {
-                ret = i915_gem_obj_ggtt_pin(reg_bo, PAGE_SIZE, true, false);
+                ret = i915_gem_obj_ggtt_pin(reg_bo, PAGE_SIZE, PIN_MAPPABLE);
                 if (ret) {
                         DRM_ERROR("failed to pin overlay register bo\n");
                         goto out_free_bo;
@@ -1386,7 +1386,7 @@ void intel_setup_overlay(struct drm_device *dev)
 
 out_unpin_bo:
         if (!OVERLAY_NEEDS_PHYSICAL(dev))
-                i915_gem_object_unpin(reg_bo);
+                i915_gem_object_ggtt_unpin(reg_bo);
 out_free_bo:
         drm_gem_object_unreference(&reg_bo->base);
 out_free:
@@ -1397,7 +1397,7 @@ out_free:
 
 void intel_cleanup_overlay(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
 
         if (!dev_priv->overlay)
                 return;
@@ -1421,7 +1421,7 @@ struct intel_overlay_error_state {
 static struct overlay_registers __iomem *
 intel_overlay_map_regs_atomic(struct intel_overlay *overlay)
 {
-        drm_i915_private_t *dev_priv = overlay->dev->dev_private;
+        struct drm_i915_private *dev_priv = overlay->dev->dev_private;
         struct overlay_registers __iomem *regs;
 
         if (OVERLAY_NEEDS_PHYSICAL(overlay->dev))
@@ -1447,7 +1447,7 @@ static void intel_overlay_unmap_regs_atomic(struct intel_overlay *overlay,
 struct intel_overlay_error_state *
 intel_overlay_capture_error_state(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_overlay *overlay = dev_priv->overlay;
         struct intel_overlay_error_state *error;
         struct overlay_registers __iomem *regs;
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index 079ea38f14d9..cb058408c70e 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -33,8 +33,6 @@
 #include <linux/moduleparam.h>
 #include "intel_drv.h"
 
-#define PCI_LBPC 0xf4 /* legacy/combination backlight modes */
-
 void
 intel_fixed_panel_mode(const struct drm_display_mode *fixed_mode,
                        struct drm_display_mode *adjusted_mode)
@@ -325,13 +323,6 @@ out:
         pipe_config->gmch_pfit.lvds_border_bits = border;
 }
 
-static int i915_panel_invert_brightness;
-MODULE_PARM_DESC(invert_brightness, "Invert backlight brightness "
-        "(-1 force normal, 0 machine defaults, 1 force inversion), please "
-        "report PCI device ID, subsystem vendor and subsystem device ID "
-        "to dri-devel@lists.freedesktop.org, if your machine needs it. "
-        "It will then be included in an upcoming module version.");
-module_param_named(invert_brightness, i915_panel_invert_brightness, int, 0600);
 static u32 intel_panel_compute_brightness(struct intel_connector *connector,
                                           u32 val)
 {
@@ -341,10 +332,10 @@ static u32 intel_panel_compute_brightness(struct intel_connector *connector,
 
         WARN_ON(panel->backlight.max == 0);
 
-        if (i915_panel_invert_brightness < 0)
+        if (i915.invert_brightness < 0)
                 return val;
 
-        if (i915_panel_invert_brightness > 0 ||
+        if (i915.invert_brightness > 0 ||
             dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS) {
                 return panel->backlight.max - val;
         }
@@ -810,13 +801,13 @@ intel_panel_detect(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         /* Assume that the BIOS does not lie through the OpRegion... */
-        if (!i915_panel_ignore_lid && dev_priv->opregion.lid_state) {
+        if (!i915.panel_ignore_lid && dev_priv->opregion.lid_state) {
                 return ioread32(dev_priv->opregion.lid_state) & 0x1 ?
                         connector_status_connected :
                         connector_status_disconnected;
         }
 
-        switch (i915_panel_ignore_lid) {
+        switch (i915.panel_ignore_lid) {
         case -2:
                 return connector_status_connected;
         case -1:
@@ -1199,9 +1190,11 @@ void intel_panel_init_backlight_funcs(struct drm_device *dev)
 }
 
 int intel_panel_init(struct intel_panel *panel,
-                     struct drm_display_mode *fixed_mode)
+                     struct drm_display_mode *fixed_mode,
+                     struct drm_display_mode *downclock_mode)
 {
         panel->fixed_mode = fixed_mode;
+        panel->downclock_mode = downclock_mode;
 
         return 0;
 }
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index e1fc35a72656..5874716774a7 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -92,12 +92,12 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_framebuffer *fb = crtc->fb;
+        struct drm_framebuffer *fb = crtc->primary->fb;
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
         struct drm_i915_gem_object *obj = intel_fb->obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int cfb_pitch;
-        int plane, i;
+        int i;
         u32 fbc_ctl;
 
         cfb_pitch = dev_priv->fbc.size / FBC_LL_SIZE;
@@ -109,7 +109,6 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc)
                 cfb_pitch = (cfb_pitch / 32) - 1;
         else
                 cfb_pitch = (cfb_pitch / 64) - 1;
-        plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
 
         /* Clear old tags */
         for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
@@ -120,7 +119,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc)
 
                 /* Set it up... */
                 fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE;
-                fbc_ctl2 |= plane;
+                fbc_ctl2 |= FBC_CTL_PLANE(intel_crtc->plane);
                 I915_WRITE(FBC_CONTROL2, fbc_ctl2);
                 I915_WRITE(FBC_FENCE_OFF, crtc->y);
         }
@@ -135,7 +134,7 @@ static void i8xx_enable_fbc(struct drm_crtc *crtc)
         fbc_ctl |= obj->fence_reg;
         I915_WRITE(FBC_CONTROL, fbc_ctl);
 
-        DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c, ",
+        DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %c\n",
                       cfb_pitch, crtc->y, plane_name(intel_crtc->plane));
 }
 
@@ -150,21 +149,23 @@ static void g4x_enable_fbc(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_framebuffer *fb = crtc->fb;
+        struct drm_framebuffer *fb = crtc->primary->fb;
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
         struct drm_i915_gem_object *obj = intel_fb->obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
         u32 dpfc_ctl;
 
-        dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
+        dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane) | DPFC_SR_EN;
+        if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
+                dpfc_ctl |= DPFC_CTL_LIMIT_2X;
+        else
+                dpfc_ctl |= DPFC_CTL_LIMIT_1X;
         dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg;
-        I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
 
         I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
 
         /* enable it... */
-        I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) | DPFC_CTL_EN);
+        I915_WRITE(DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
 
         DRM_DEBUG_KMS("enabled fbc on plane %c\n", plane_name(intel_crtc->plane));
 }
@@ -220,22 +221,20 @@ static void ironlake_enable_fbc(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_framebuffer *fb = crtc->fb;
+        struct drm_framebuffer *fb = crtc->primary->fb;
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
         struct drm_i915_gem_object *obj = intel_fb->obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
         u32 dpfc_ctl;
 
-        dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
-        dpfc_ctl &= DPFC_RESERVED;
-        dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X);
-        /* Set persistent mode for front-buffer rendering, ala X. */
-        dpfc_ctl |= DPFC_CTL_PERSISTENT_MODE;
+        dpfc_ctl = DPFC_CTL_PLANE(intel_crtc->plane);
+        if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
+                dpfc_ctl |= DPFC_CTL_LIMIT_2X;
+        else
+                dpfc_ctl |= DPFC_CTL_LIMIT_1X;
         dpfc_ctl |= DPFC_CTL_FENCE_EN;
         if (IS_GEN5(dev))
                 dpfc_ctl |= obj->fence_reg;
-        I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
 
         I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
         I915_WRITE(ILK_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj) | ILK_FBC_RT_VALID);
@@ -278,24 +277,31 @@ static void gen7_enable_fbc(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_framebuffer *fb = crtc->fb;
+        struct drm_framebuffer *fb = crtc->primary->fb;
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
         struct drm_i915_gem_object *obj = intel_fb->obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        u32 dpfc_ctl;
 
-        I915_WRITE(IVB_FBC_RT_BASE, i915_gem_obj_ggtt_offset(obj));
+        dpfc_ctl = IVB_DPFC_CTL_PLANE(intel_crtc->plane);
+        if (drm_format_plane_cpp(fb->pixel_format, 0) == 2)
+                dpfc_ctl |= DPFC_CTL_LIMIT_2X;
+        else
+                dpfc_ctl |= DPFC_CTL_LIMIT_1X;
+        dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN;
 
-        I915_WRITE(ILK_DPFC_CONTROL, DPFC_CTL_EN | DPFC_CTL_LIMIT_1X |
-                   IVB_DPFC_CTL_FENCE_EN |
-                   intel_crtc->plane << IVB_DPFC_CTL_PLANE_SHIFT);
+        I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
 
         if (IS_IVYBRIDGE(dev)) {
                 /* WaFbcAsynchFlipDisableFbcQueue:ivb */
-                I915_WRITE(ILK_DISPLAY_CHICKEN1, ILK_FBCQ_DIS);
+                I915_WRITE(ILK_DISPLAY_CHICKEN1,
+                           I915_READ(ILK_DISPLAY_CHICKEN1) |
+                           ILK_FBCQ_DIS);
         } else {
-                /* WaFbcAsynchFlipDisableFbcQueue:hsw */
-                I915_WRITE(HSW_PIPE_SLICE_CHICKEN_1(intel_crtc->pipe),
-                           HSW_BYPASS_FBC_QUEUE);
+                /* WaFbcAsynchFlipDisableFbcQueue:hsw,bdw */
+                I915_WRITE(CHICKEN_PIPESL_1(intel_crtc->pipe),
+                           I915_READ(CHICKEN_PIPESL_1(intel_crtc->pipe)) |
+                           HSW_FBCQ_DIS);
         }
 
         I915_WRITE(SNB_DPFC_CTL_SA,
@@ -330,11 +336,11 @@ static void intel_fbc_work_fn(struct work_struct *__work)
                 /* Double check that we haven't switched fb without cancelling
                  * the prior work.
                  */
-                if (work->crtc->fb == work->fb) {
+                if (work->crtc->primary->fb == work->fb) {
                         dev_priv->display.enable_fbc(work->crtc);
 
                         dev_priv->fbc.plane = to_intel_crtc(work->crtc)->plane;
-                        dev_priv->fbc.fb_id = work->crtc->fb->base.id;
+                        dev_priv->fbc.fb_id = work->crtc->primary->fb->base.id;
                         dev_priv->fbc.y = work->crtc->y;
                 }
 
@@ -387,7 +393,7 @@ static void intel_enable_fbc(struct drm_crtc *crtc)
         }
 
         work->crtc = crtc;
-        work->fb = crtc->fb;
+        work->fb = crtc->primary->fb;
         INIT_DELAYED_WORK(&work->work, intel_fbc_work_fn);
 
         dev_priv->fbc.fbc_work = work;
@@ -466,7 +472,7 @@ void intel_update_fbc(struct drm_device *dev)
                 return;
         }
 
-        if (!i915_powersave) {
+        if (!i915.powersave) {
                 if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
                         DRM_DEBUG_KMS("fbc disabled per module param\n");
                 return;
@@ -493,25 +499,25 @@ void intel_update_fbc(struct drm_device *dev)
                 }
         }
 
-        if (!crtc || crtc->fb == NULL) {
+        if (!crtc || crtc->primary->fb == NULL) {
                 if (set_no_fbc_reason(dev_priv, FBC_NO_OUTPUT))
                         DRM_DEBUG_KMS("no output, disabling\n");
                 goto out_disable;
         }
 
         intel_crtc = to_intel_crtc(crtc);
-        fb = crtc->fb;
+        fb = crtc->primary->fb;
         intel_fb = to_intel_framebuffer(fb);
         obj = intel_fb->obj;
         adjusted_mode = &intel_crtc->config.adjusted_mode;
 
-        if (i915_enable_fbc < 0 &&
+        if (i915.enable_fbc < 0 &&
             INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev)) {
                 if (set_no_fbc_reason(dev_priv, FBC_CHIP_DEFAULT))
                         DRM_DEBUG_KMS("disabled per chip default\n");
                 goto out_disable;
         }
-        if (!i915_enable_fbc) {
+        if (!i915.enable_fbc) {
                 if (set_no_fbc_reason(dev_priv, FBC_MODULE_PARAM))
                         DRM_DEBUG_KMS("fbc disabled per module param\n");
                 goto out_disable;
@@ -537,7 +543,7 @@ void intel_update_fbc(struct drm_device *dev)
                         DRM_DEBUG_KMS("mode too large for compression, disabling\n");
                 goto out_disable;
         }
-        if ((INTEL_INFO(dev)->gen < 4 || IS_HASWELL(dev)) &&
+        if ((INTEL_INFO(dev)->gen < 4 || HAS_DDI(dev)) &&
             intel_crtc->plane != PLANE_A) {
                 if (set_no_fbc_reason(dev_priv, FBC_BAD_PLANE))
                         DRM_DEBUG_KMS("plane not A, disabling compression\n");
@@ -617,7 +623,7 @@ out_disable:
 
 static void i915_pineview_get_mem_freq(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 tmp;
 
         tmp = I915_READ(CLKCFG);
@@ -656,7 +662,7 @@ static void i915_pineview_get_mem_freq(struct drm_device *dev)
 
 static void i915_ironlake_get_mem_freq(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         u16 ddrpll, csipll;
 
         ddrpll = I915_READ16(DDRMPLL1);
@@ -1035,7 +1041,7 @@ static void pineview_update_wm(struct drm_crtc *unused_crtc)
         crtc = single_enabled_crtc(dev);
         if (crtc) {
                 const struct drm_display_mode *adjusted_mode;
-                int pixel_size = crtc->fb->bits_per_pixel / 8;
+                int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
                 int clock;
 
                 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
@@ -1115,7 +1121,7 @@ static bool g4x_compute_wm0(struct drm_device *dev,
         clock = adjusted_mode->crtc_clock;
         htotal = adjusted_mode->crtc_htotal;
         hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
-        pixel_size = crtc->fb->bits_per_pixel / 8;
+        pixel_size = crtc->primary->fb->bits_per_pixel / 8;
 
         /* Use the small buffer method to calculate plane watermark */
         entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
@@ -1128,9 +1134,9 @@ static bool g4x_compute_wm0(struct drm_device *dev,
                 *plane_wm = display->max_wm;
 
         /* Use the large buffer method to calculate cursor watermark */
-        line_time_us = ((htotal * 1000) / clock);
+        line_time_us = max(htotal * 1000 / clock, 1);
         line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
-        entries = line_count * 64 * pixel_size;
+        entries = line_count * to_intel_crtc(crtc)->cursor_width * pixel_size;
         tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
         if (tlb_miss > 0)
                 entries += tlb_miss;
@@ -1202,9 +1208,9 @@ static bool g4x_compute_srwm(struct drm_device *dev,
         clock = adjusted_mode->crtc_clock;
         htotal = adjusted_mode->crtc_htotal;
         hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
-        pixel_size = crtc->fb->bits_per_pixel / 8;
+        pixel_size = crtc->primary->fb->bits_per_pixel / 8;
 
-        line_time_us = (htotal * 1000) / clock;
+        line_time_us = max(htotal * 1000 / clock, 1);
         line_count = (latency_ns / line_time_us + 1000) / 1000;
         line_size = hdisplay * pixel_size;
 
@@ -1216,7 +1222,7 @@ static bool g4x_compute_srwm(struct drm_device *dev,
         *display_wm = entries + display->guard_size;
 
         /* calculate the self-refresh watermark for display cursor */
-        entries = line_count * pixel_size * 64;
+        entries = line_count * pixel_size * to_intel_crtc(crtc)->cursor_width;
         entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
         *cursor_wm = entries + cursor->guard_size;
 
@@ -1241,7 +1247,7 @@ static bool vlv_compute_drain_latency(struct drm_device *dev,
                 return false;
 
         clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock;
-        pixel_size = crtc->fb->bits_per_pixel / 8;      /* BPP */
+        pixel_size = crtc->primary->fb->bits_per_pixel / 8;     /* BPP */
 
         entries = (clock / 1000) * pixel_size;
         *plane_prec_mult = (entries > 256) ?
@@ -1433,11 +1439,11 @@ static void i965_update_wm(struct drm_crtc *unused_crtc)
                 int clock = adjusted_mode->crtc_clock;
                 int htotal = adjusted_mode->crtc_htotal;
                 int hdisplay = to_intel_crtc(crtc)->config.pipe_src_w;
-                int pixel_size = crtc->fb->bits_per_pixel / 8;
+                int pixel_size = crtc->primary->fb->bits_per_pixel / 8;
                 unsigned long line_time_us;
                 int entries;
 
-                line_time_us = ((htotal * 1000) / clock);
+                line_time_us = max(htotal * 1000 / clock, 1);
 
                 /* Use ns/us then divide to preserve precision */
                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
@@ -1451,7 +1457,7 @@ static void i965_update_wm(struct drm_crtc *unused_crtc)
                               entries, srwm);
 
                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
-                        pixel_size * 64;
+                        pixel_size * to_intel_crtc(crtc)->cursor_width;
                 entries = DIV_ROUND_UP(entries,
                                           i965_cursor_wm_info.cacheline_size);
                 cursor_sr = i965_cursor_wm_info.fifo_size -
@@ -1506,7 +1512,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
         crtc = intel_get_crtc_for_plane(dev, 0);
         if (intel_crtc_active(crtc)) {
                 const struct drm_display_mode *adjusted_mode;
-                int cpp = crtc->fb->bits_per_pixel / 8;
+                int cpp = crtc->primary->fb->bits_per_pixel / 8;
                 if (IS_GEN2(dev))
                         cpp = 4;
 
@@ -1522,7 +1528,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
         crtc = intel_get_crtc_for_plane(dev, 1);
         if (intel_crtc_active(crtc)) {
                 const struct drm_display_mode *adjusted_mode;
-                int cpp = crtc->fb->bits_per_pixel / 8;
+                int cpp = crtc->primary->fb->bits_per_pixel / 8;
                 if (IS_GEN2(dev))
                         cpp = 4;
 
@@ -1559,11 +1565,11 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
                 int clock = adjusted_mode->crtc_clock;
                 int htotal = adjusted_mode->crtc_htotal;
                 int hdisplay = to_intel_crtc(enabled)->config.pipe_src_w;
-                int pixel_size = enabled->fb->bits_per_pixel / 8;
+                int pixel_size = enabled->primary->fb->bits_per_pixel / 8;
                 unsigned long line_time_us;
                 int entries;
 
-                line_time_us = (htotal * 1000) / clock;
+                line_time_us = max(htotal * 1000 / clock, 1);
 
                 /* Use ns/us then divide to preserve precision */
                 entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
@@ -1886,7 +1892,7 @@ static unsigned int ilk_cursor_wm_max(const struct drm_device *dev,
 }
 
 /* Calculate the maximum FBC watermark */
-static unsigned int ilk_fbc_wm_max(struct drm_device *dev)
+static unsigned int ilk_fbc_wm_max(const struct drm_device *dev)
 {
         /* max that registers can hold */
         if (INTEL_INFO(dev)->gen >= 8)
@@ -1895,7 +1901,7 @@ static unsigned int ilk_fbc_wm_max(struct drm_device *dev)
                 return 15;
 }
 
-static void ilk_compute_wm_maximums(struct drm_device *dev,
+static void ilk_compute_wm_maximums(const struct drm_device *dev,
                                     int level,
                                     const struct intel_wm_config *config,
                                     enum intel_ddb_partitioning ddb_partitioning,
@@ -1948,7 +1954,7 @@ static bool ilk_validate_wm_level(int level,
         return ret;
 }
 
-static void ilk_compute_wm_level(struct drm_i915_private *dev_priv,
+static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
                                  int level,
                                  const struct ilk_pipe_wm_parameters *p,
                                  struct intel_wm_level *result)
@@ -2079,7 +2085,7 @@ static void intel_print_wm_latency(struct drm_device *dev,
         }
 }
 
-static void intel_setup_wm_latency(struct drm_device *dev)
+static void ilk_setup_wm_latency(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
@@ -2111,10 +2117,10 @@ static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
         if (p->active) {
                 p->pipe_htotal = intel_crtc->config.adjusted_mode.crtc_htotal;
                 p->pixel_rate = ilk_pipe_pixel_rate(dev, crtc);
-                p->pri.bytes_per_pixel = crtc->fb->bits_per_pixel / 8;
+                p->pri.bytes_per_pixel = crtc->primary->fb->bits_per_pixel / 8;
                 p->cur.bytes_per_pixel = 4;
                 p->pri.horiz_pixels = intel_crtc->config.pipe_src_w;
-                p->cur.horiz_pixels = 64;
+                p->cur.horiz_pixels = intel_crtc->cursor_width;
                 /* TODO: for now, assume primary and cursor planes are always enabled. */
                 p->pri.enabled = true;
                 p->cur.enabled = true;
@@ -2123,7 +2129,7 @@ static void ilk_compute_wm_parameters(struct drm_crtc *crtc,
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
                 config->num_pipes_active += intel_crtc_active(crtc);
 
-        list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
+        drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) {
                 struct intel_plane *intel_plane = to_intel_plane(plane);
 
                 if (intel_plane->pipe == pipe)
@@ -2140,7 +2146,7 @@ static bool intel_compute_pipe_wm(struct drm_crtc *crtc,
                                   struct intel_pipe_wm *pipe_wm)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        const struct drm_i915_private *dev_priv = dev->dev_private;
         int level, max_level = ilk_wm_max_level(dev);
         /* LP0 watermark maximums depend on this pipe alone */
         struct intel_wm_config config = {
@@ -2738,7 +2744,7 @@ intel_alloc_context_page(struct drm_device *dev)
                 return NULL;
         }
 
-        ret = i915_gem_obj_ggtt_pin(ctx, 4096, true, false);
+        ret = i915_gem_obj_ggtt_pin(ctx, 4096, 0);
         if (ret) {
                 DRM_ERROR("failed to pin power context: %d\n", ret);
                 goto err_unref;
@@ -2753,7 +2759,7 @@ intel_alloc_context_page(struct drm_device *dev)
         return ctx;
 
 err_unpin:
-        i915_gem_object_unpin(ctx);
+        i915_gem_object_ggtt_unpin(ctx);
 err_unref:
         drm_gem_object_unreference(&ctx->base);
         return NULL;
@@ -2901,9 +2907,9 @@ static u32 gen6_rps_limits(struct drm_i915_private *dev_priv, u8 val)
          * the hw runs at the minimal clock before selecting the desired
          * frequency, if the down threshold expires in that window we will not
          * receive a down interrupt. */
-        limits = dev_priv->rps.max_delay << 24;
-        if (val <= dev_priv->rps.min_delay)
-                limits |= dev_priv->rps.min_delay << 16;
+        limits = dev_priv->rps.max_freq_softlimit << 24;
+        if (val <= dev_priv->rps.min_freq_softlimit)
+                limits |= dev_priv->rps.min_freq_softlimit << 16;
 
         return limits;
 }
@@ -2915,26 +2921,26 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
         new_power = dev_priv->rps.power;
         switch (dev_priv->rps.power) {
         case LOW_POWER:
-                if (val > dev_priv->rps.rpe_delay + 1 && val > dev_priv->rps.cur_delay)
+                if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
                         new_power = BETWEEN;
                 break;
 
         case BETWEEN:
-                if (val <= dev_priv->rps.rpe_delay && val < dev_priv->rps.cur_delay)
+                if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
                         new_power = LOW_POWER;
-                else if (val >= dev_priv->rps.rp0_delay && val > dev_priv->rps.cur_delay)
+                else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
                         new_power = HIGH_POWER;
                 break;
 
         case HIGH_POWER:
-                if (val < (dev_priv->rps.rp1_delay + dev_priv->rps.rp0_delay) >> 1 && val < dev_priv->rps.cur_delay)
+                if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
                         new_power = BETWEEN;
                 break;
         }
         /* Max/min bins are special */
-        if (val == dev_priv->rps.min_delay)
+        if (val == dev_priv->rps.min_freq_softlimit)
                 new_power = LOW_POWER;
-        if (val == dev_priv->rps.max_delay)
+        if (val == dev_priv->rps.max_freq_softlimit)
                 new_power = HIGH_POWER;
         if (new_power == dev_priv->rps.power)
                 return;
@@ -3000,41 +3006,113 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
         dev_priv->rps.last_adj = 0;
 }
 
+static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
+{
+        u32 mask = 0;
+
+        if (val > dev_priv->rps.min_freq_softlimit)
+                mask |= GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT;
+        if (val < dev_priv->rps.max_freq_softlimit)
+                mask |= GEN6_PM_RP_UP_THRESHOLD;
+
+        /* IVB and SNB hard hangs on looping batchbuffer
+         * if GEN6_PM_UP_EI_EXPIRED is masked.
+         */
+        if (INTEL_INFO(dev_priv->dev)->gen <= 7 && !IS_HASWELL(dev_priv->dev))
+                mask |= GEN6_PM_RP_UP_EI_EXPIRED;
+
+        return ~mask;
+}
+
+/* gen6_set_rps is called to update the frequency request, but should also be
+ * called when the range (min_delay and max_delay) is modified so that we can
+ * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
 void gen6_set_rps(struct drm_device *dev, u8 val)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
-        WARN_ON(val > dev_priv->rps.max_delay);
-        WARN_ON(val < dev_priv->rps.min_delay);
-
-        if (val == dev_priv->rps.cur_delay)
-                return;
+        WARN_ON(val > dev_priv->rps.max_freq_softlimit);
+        WARN_ON(val < dev_priv->rps.min_freq_softlimit);
 
-        gen6_set_rps_thresholds(dev_priv, val);
+        /* min/max delay may still have been modified so be sure to
+         * write the limits value.
+         */
+        if (val != dev_priv->rps.cur_freq) {
+                gen6_set_rps_thresholds(dev_priv, val);
 
-        if (IS_HASWELL(dev))
-                I915_WRITE(GEN6_RPNSWREQ,
-                           HSW_FREQUENCY(val));
-        else
-                I915_WRITE(GEN6_RPNSWREQ,
-                           GEN6_FREQUENCY(val) |
-                           GEN6_OFFSET(0) |
-                           GEN6_AGGRESSIVE_TURBO);
+                if (IS_HASWELL(dev))
+                        I915_WRITE(GEN6_RPNSWREQ,
+                                   HSW_FREQUENCY(val));
+                else
+                        I915_WRITE(GEN6_RPNSWREQ,
+                                   GEN6_FREQUENCY(val) |
+                                   GEN6_OFFSET(0) |
+                                   GEN6_AGGRESSIVE_TURBO);
+        }
 
         /* Make sure we continue to get interrupts
          * until we hit the minimum or maximum frequencies.
          */
-        I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
-                   gen6_rps_limits(dev_priv, val));
+        I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, gen6_rps_limits(dev_priv, val));
+        I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
 
         POSTING_READ(GEN6_RPNSWREQ);
 
-        dev_priv->rps.cur_delay = val;
-
+        dev_priv->rps.cur_freq = val;
         trace_intel_gpu_freq_change(val * 50);
 }
 
+/* vlv_set_rps_idle: Set the frequency to Rpn if Gfx clocks are down
+ *
+ * * If Gfx is Idle, then
+ * 1. Mask Turbo interrupts
+ * 2. Bring up Gfx clock
+ * 3. Change the freq to Rpn and wait till P-Unit updates freq
+ * 4. Clear the Force GFX CLK ON bit so that Gfx can down
+ * 5. Unmask Turbo interrupts
+*/
+static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
+{
+        /*
+         * When we are idle.  Drop to min voltage state.
+         */
+
+        if (dev_priv->rps.cur_freq <= dev_priv->rps.min_freq_softlimit)
+                return;
+
+        /* Mask turbo interrupt so that they will not come in between */
+        I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
+
+        /* Bring up the Gfx clock */
+        I915_WRITE(VLV_GTLC_SURVIVABILITY_REG,
+                I915_READ(VLV_GTLC_SURVIVABILITY_REG) |
+                                VLV_GFX_CLK_FORCE_ON_BIT);
+
+        if (wait_for(((VLV_GFX_CLK_STATUS_BIT &
+                I915_READ(VLV_GTLC_SURVIVABILITY_REG)) != 0), 5)) {
+                        DRM_ERROR("GFX_CLK_ON request timed out\n");
+                return;
+        }
+
+        dev_priv->rps.cur_freq = dev_priv->rps.min_freq_softlimit;
+
+        vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ,
+                                        dev_priv->rps.min_freq_softlimit);
+
+        if (wait_for(((vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS))
+                                & GENFREQSTATUS) == 0, 5))
+                DRM_ERROR("timed out waiting for Punit\n");
+
+        /* Release the Gfx clock */
+        I915_WRITE(VLV_GTLC_SURVIVABILITY_REG,
+                I915_READ(VLV_GTLC_SURVIVABILITY_REG) &
+                                ~VLV_GFX_CLK_FORCE_ON_BIT);
+
+        I915_WRITE(GEN6_PMINTRMSK,
+                   gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
+}
+
 void gen6_rps_idle(struct drm_i915_private *dev_priv)
 {
         struct drm_device *dev = dev_priv->dev;
@@ -3042,9 +3120,9 @@ void gen6_rps_idle(struct drm_i915_private *dev_priv)
         mutex_lock(&dev_priv->rps.hw_lock);
         if (dev_priv->rps.enabled) {
                 if (IS_VALLEYVIEW(dev))
-                        valleyview_set_rps(dev_priv->dev, dev_priv->rps.min_delay);
+                        vlv_set_rps_idle(dev_priv);
                 else
-                        gen6_set_rps(dev_priv->dev, dev_priv->rps.min_delay);
+                        gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
                 dev_priv->rps.last_adj = 0;
         }
         mutex_unlock(&dev_priv->rps.hw_lock);
@@ -3057,9 +3135,9 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv)
         mutex_lock(&dev_priv->rps.hw_lock);
         if (dev_priv->rps.enabled) {
                 if (IS_VALLEYVIEW(dev))
-                        valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_delay);
+                        valleyview_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
                 else
-                        gen6_set_rps(dev_priv->dev, dev_priv->rps.max_delay);
+                        gen6_set_rps(dev_priv->dev, dev_priv->rps.max_freq_softlimit);
                 dev_priv->rps.last_adj = 0;
         }
         mutex_unlock(&dev_priv->rps.hw_lock);
@@ -3070,21 +3148,20 @@ void valleyview_set_rps(struct drm_device *dev, u8 val)
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
-        WARN_ON(val > dev_priv->rps.max_delay);
-        WARN_ON(val < dev_priv->rps.min_delay);
+        WARN_ON(val > dev_priv->rps.max_freq_softlimit);
+        WARN_ON(val < dev_priv->rps.min_freq_softlimit);
 
         DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n",
-                         vlv_gpu_freq(dev_priv, dev_priv->rps.cur_delay),
-                         dev_priv->rps.cur_delay,
+                         vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
+                         dev_priv->rps.cur_freq,
                          vlv_gpu_freq(dev_priv, val), val);
 
-        if (val == dev_priv->rps.cur_delay)
-                return;
-
-        vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
+        if (val != dev_priv->rps.cur_freq)
+                vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
 
-        dev_priv->rps.cur_delay = val;
+        I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
 
+        dev_priv->rps.cur_freq = val;
         trace_intel_gpu_freq_change(vlv_gpu_freq(dev_priv, val));
 }
 
@@ -3093,7 +3170,8 @@ static void gen6_disable_rps_interrupts(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
-        I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) & ~GEN6_PM_RPS_EVENTS);
+        I915_WRITE(GEN6_PMIER, I915_READ(GEN6_PMIER) &
+                                ~dev_priv->pm_rps_events);
         /* Complete PM interrupt masking here doesn't race with the rps work
          * item again unmasking PM interrupts because that is using a different
          * register (PMIMR) to mask PM interrupts. The only risk is in leaving
@@ -3103,7 +3181,7 @@ static void gen6_disable_rps_interrupts(struct drm_device *dev)
         dev_priv->rps.pm_iir = 0;
         spin_unlock_irq(&dev_priv->irq_lock);
 
-        I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS);
+        I915_WRITE(GEN6_PMIIR, dev_priv->pm_rps_events);
 }
 
 static void gen6_disable_rps(struct drm_device *dev)
@@ -3123,25 +3201,14 @@ static void valleyview_disable_rps(struct drm_device *dev)
         I915_WRITE(GEN6_RC_CONTROL, 0);
 
         gen6_disable_rps_interrupts(dev);
-
-        if (dev_priv->vlv_pctx) {
-                drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
-                dev_priv->vlv_pctx = NULL;
-        }
 }
 
 static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
 {
-        if (IS_GEN6(dev))
-                DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
-
-        if (IS_HASWELL(dev))
-                DRM_DEBUG_DRIVER("Haswell: only RC6 available\n");
-
         DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
-                        (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
-                        (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
-                        (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
+                 (mode & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off",
+                 (mode & GEN6_RC_CTL_RC6p_ENABLE) ? "on" : "off",
+                 (mode & GEN6_RC_CTL_RC6pp_ENABLE) ? "on" : "off");
 }
 
 int intel_enable_rc6(const struct drm_device *dev)
@@ -3151,44 +3218,28 @@ int intel_enable_rc6(const struct drm_device *dev)
                 return 0;
 
         /* Respect the kernel parameter if it is set */
-        if (i915_enable_rc6 >= 0)
-                return i915_enable_rc6;
+        if (i915.enable_rc6 >= 0)
+                return i915.enable_rc6;
 
         /* Disable RC6 on Ironlake */
         if (INTEL_INFO(dev)->gen == 5)
                 return 0;
 
-        if (IS_HASWELL(dev))
-                return INTEL_RC6_ENABLE;
-
-        /* snb/ivb have more than one rc6 state. */
-        if (INTEL_INFO(dev)->gen == 6)
-                return INTEL_RC6_ENABLE;
+        if (IS_IVYBRIDGE(dev))
+                return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
 
-        return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
+        return INTEL_RC6_ENABLE;
 }
 
 static void gen6_enable_rps_interrupts(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 enabled_intrs;
 
         spin_lock_irq(&dev_priv->irq_lock);
         WARN_ON(dev_priv->rps.pm_iir);
-        snb_enable_pm_irq(dev_priv, GEN6_PM_RPS_EVENTS);
-        I915_WRITE(GEN6_PMIIR, GEN6_PM_RPS_EVENTS);
+        snb_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
+        I915_WRITE(GEN6_PMIIR, dev_priv->pm_rps_events);
         spin_unlock_irq(&dev_priv->irq_lock);
-
-        /* only unmask PM interrupts we need. Mask all others. */
-        enabled_intrs = GEN6_PM_RPS_EVENTS;
-
-        /* IVB and SNB hard hangs on looping batchbuffer
-         * if GEN6_PM_UP_EI_EXPIRED is masked.
-         */
-        if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
-                enabled_intrs |= GEN6_PM_RP_UP_EI_EXPIRED;
-
-        I915_WRITE(GEN6_PMINTRMSK, ~enabled_intrs);
 }
 
 static void gen8_enable_rps(struct drm_device *dev)
@@ -3222,10 +3273,10 @@ static void gen8_enable_rps(struct drm_device *dev)
         /* 3: Enable RC6 */
         if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
                 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
-        DRM_INFO("RC6 %s\n", (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ? "on" : "off");
+        intel_print_rc6_info(dev, rc6_mask);
         I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
-                        GEN6_RC_CTL_EI_MODE(1) |
-                        rc6_mask);
+                                    GEN6_RC_CTL_EI_MODE(1) |
+                                    rc6_mask);
 
         /* 4 Program defaults and thresholds for RPS*/
         I915_WRITE(GEN6_RPNSWREQ, HSW_FREQUENCY(10)); /* Request 500 MHz */
@@ -3235,8 +3286,8 @@ static void gen8_enable_rps(struct drm_device *dev)
 
         /* Docs recommend 900MHz, and 300 MHz respectively */
         I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
-                   dev_priv->rps.max_delay << 24 |
-                   dev_priv->rps.min_delay << 16);
+                   dev_priv->rps.max_freq_softlimit << 24 |
+                   dev_priv->rps.min_freq_softlimit << 16);
 
         I915_WRITE(GEN6_RP_UP_THRESHOLD, 7600000 / 128); /* 76ms busyness per EI, 90% */
         I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 31300000 / 128); /* 313ms busyness per EI, 70%*/
@@ -3269,7 +3320,7 @@ static void gen6_enable_rps(struct drm_device *dev)
         struct intel_ring_buffer *ring;
         u32 rp_state_cap;
         u32 gt_perf_status;
-        u32 rc6vids, pcu_mbox, rc6_mask = 0;
+        u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
         u32 gtfifodbg;
         int rc6_mode;
         int i, ret;
@@ -3295,13 +3346,23 @@ static void gen6_enable_rps(struct drm_device *dev)
         rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
         gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
 
-        /* In units of 50MHz */
-        dev_priv->rps.hw_max = dev_priv->rps.max_delay = rp_state_cap & 0xff;
-        dev_priv->rps.min_delay = (rp_state_cap >> 16) & 0xff;
-        dev_priv->rps.rp1_delay = (rp_state_cap >>  8) & 0xff;
-        dev_priv->rps.rp0_delay = (rp_state_cap >>  0) & 0xff;
-        dev_priv->rps.rpe_delay = dev_priv->rps.rp1_delay;
-        dev_priv->rps.cur_delay = 0;
+        /* All of these values are in units of 50MHz */
+        dev_priv->rps.cur_freq          = 0;
+        /* static values from HW: RP0 < RPe < RP1 < RPn (min_freq) */
+        dev_priv->rps.rp1_freq          = (rp_state_cap >>  8) & 0xff;
+        dev_priv->rps.rp0_freq          = (rp_state_cap >>  0) & 0xff;
+        dev_priv->rps.min_freq          = (rp_state_cap >> 16) & 0xff;
+        /* XXX: only BYT has a special efficient freq */
+        dev_priv->rps.efficient_freq    = dev_priv->rps.rp1_freq;
+        /* hw_max = RP0 until we check for overclocking */
+        dev_priv->rps.max_freq          = dev_priv->rps.rp0_freq;
+
+        /* Preserve min/max settings in case of re-init */
+        if (dev_priv->rps.max_freq_softlimit == 0)
+                dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
+
+        if (dev_priv->rps.min_freq_softlimit == 0)
+                dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
 
         /* disable the counters and set deterministic thresholds */
         I915_WRITE(GEN6_RC_CONTROL, 0);
@@ -3350,21 +3411,19 @@ static void gen6_enable_rps(struct drm_device *dev)
         I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
 
         ret = sandybridge_pcode_write(dev_priv, GEN6_PCODE_WRITE_MIN_FREQ_TABLE, 0);
-        if (!ret) {
-                pcu_mbox = 0;
-                ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
-                if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
-                        DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
-                                         (dev_priv->rps.max_delay & 0xff) * 50,
-                                         (pcu_mbox & 0xff) * 50);
-                        dev_priv->rps.hw_max = pcu_mbox & 0xff;
-                }
-        } else {
+        if (ret)
                 DRM_DEBUG_DRIVER("Failed to set the min frequency\n");
+
+        ret = sandybridge_pcode_read(dev_priv, GEN6_READ_OC_PARAMS, &pcu_mbox);
+        if (!ret && (pcu_mbox & (1<<31))) { /* OC supported */
+                DRM_DEBUG_DRIVER("Overclocking supported. Max: %dMHz, Overclock max: %dMHz\n",
+                                 (dev_priv->rps.max_freq_softlimit & 0xff) * 50,
+                                 (pcu_mbox & 0xff) * 50);
+                dev_priv->rps.max_freq = pcu_mbox & 0xff;
         }
 
         dev_priv->rps.power = HIGH_POWER; /* force a reset */
-        gen6_set_rps(dev_priv->dev, dev_priv->rps.min_delay);
+        gen6_set_rps(dev_priv->dev, dev_priv->rps.min_freq_softlimit);
 
         gen6_enable_rps_interrupts(dev);
 
@@ -3420,9 +3479,9 @@ void gen6_update_ring_freq(struct drm_device *dev)
          * to use for memory access.  We do this by specifying the IA frequency
          * the PCU should use as a reference to determine the ring frequency.
          */
-        for (gpu_freq = dev_priv->rps.max_delay; gpu_freq >= dev_priv->rps.min_delay;
+        for (gpu_freq = dev_priv->rps.max_freq_softlimit; gpu_freq >= dev_priv->rps.min_freq_softlimit;
              gpu_freq--) {
-                int diff = dev_priv->rps.max_delay - gpu_freq;
+                int diff = dev_priv->rps.max_freq_softlimit - gpu_freq;
                 unsigned int ia_freq = 0, ring_freq = 0;
 
                 if (INTEL_INFO(dev)->gen >= 8) {
@@ -3485,6 +3544,15 @@ int valleyview_rps_min_freq(struct drm_i915_private *dev_priv)
         return vlv_punit_read(dev_priv, PUNIT_REG_GPU_LFM) & 0xff;
 }
 
+/* Check that the pctx buffer wasn't move under us. */
+static void valleyview_check_pctx(struct drm_i915_private *dev_priv)
+{
+        unsigned long pctx_addr = I915_READ(VLV_PCBR) & ~4095;
+
+        WARN_ON(pctx_addr != dev_priv->mm.stolen_base +
+                             dev_priv->vlv_pctx->stolen->start);
+}
+
 static void valleyview_setup_pctx(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -3529,6 +3597,17 @@ out:
         dev_priv->vlv_pctx = pctx;
 }
 
+static void valleyview_cleanup_pctx(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (WARN_ON(!dev_priv->vlv_pctx))
+                return;
+
+        drm_gem_object_unreference(&dev_priv->vlv_pctx->base);
+        dev_priv->vlv_pctx = NULL;
+}
+
 static void valleyview_enable_rps(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -3538,6 +3617,8 @@ static void valleyview_enable_rps(struct drm_device *dev)
 
         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
 
+        valleyview_check_pctx(dev_priv);
+
         if ((gtfifodbg = I915_READ(GTFIFODBG))) {
                 DRM_DEBUG_DRIVER("GT fifo had a previous error %x\n",
                                  gtfifodbg);
@@ -3588,32 +3669,39 @@ static void valleyview_enable_rps(struct drm_device *dev)
         DRM_DEBUG_DRIVER("GPLL enabled? %s\n", val & 0x10 ? "yes" : "no");
         DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val);
 
-        dev_priv->rps.cur_delay = (val >> 8) & 0xff;
+        dev_priv->rps.cur_freq = (val >> 8) & 0xff;
         DRM_DEBUG_DRIVER("current GPU freq: %d MHz (%u)\n",
-                         vlv_gpu_freq(dev_priv, dev_priv->rps.cur_delay),
-                         dev_priv->rps.cur_delay);
+                         vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
+                         dev_priv->rps.cur_freq);
 
-        dev_priv->rps.max_delay = valleyview_rps_max_freq(dev_priv);
-        dev_priv->rps.hw_max = dev_priv->rps.max_delay;
+        dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
+        dev_priv->rps.rp0_freq  = dev_priv->rps.max_freq;
         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
-                         vlv_gpu_freq(dev_priv, dev_priv->rps.max_delay),
-                         dev_priv->rps.max_delay);
+                         vlv_gpu_freq(dev_priv, dev_priv->rps.max_freq),
+                         dev_priv->rps.max_freq);
 
-        dev_priv->rps.rpe_delay = valleyview_rps_rpe_freq(dev_priv);
+        dev_priv->rps.efficient_freq = valleyview_rps_rpe_freq(dev_priv);
         DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n",
-                         vlv_gpu_freq(dev_priv, dev_priv->rps.rpe_delay),
-                         dev_priv->rps.rpe_delay);
+                         vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
+                         dev_priv->rps.efficient_freq);
 
-        dev_priv->rps.min_delay = valleyview_rps_min_freq(dev_priv);
+        dev_priv->rps.min_freq = valleyview_rps_min_freq(dev_priv);
         DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n",
-                         vlv_gpu_freq(dev_priv, dev_priv->rps.min_delay),
-                         dev_priv->rps.min_delay);
+                         vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq),
+                         dev_priv->rps.min_freq);
+
+        /* Preserve min/max settings in case of re-init */
+        if (dev_priv->rps.max_freq_softlimit == 0)
+                dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
+
+        if (dev_priv->rps.min_freq_softlimit == 0)
+                dev_priv->rps.min_freq_softlimit = dev_priv->rps.min_freq;
 
         DRM_DEBUG_DRIVER("setting GPU freq to %d MHz (%u)\n",
-                         vlv_gpu_freq(dev_priv, dev_priv->rps.rpe_delay),
-                         dev_priv->rps.rpe_delay);
+                         vlv_gpu_freq(dev_priv, dev_priv->rps.efficient_freq),
+                         dev_priv->rps.efficient_freq);
 
-        valleyview_set_rps(dev_priv->dev, dev_priv->rps.rpe_delay);
+        valleyview_set_rps(dev_priv->dev, dev_priv->rps.efficient_freq);
 
         gen6_enable_rps_interrupts(dev);
 
@@ -3625,13 +3713,13 @@ void ironlake_teardown_rc6(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         if (dev_priv->ips.renderctx) {
-                i915_gem_object_unpin(dev_priv->ips.renderctx);
+                i915_gem_object_ggtt_unpin(dev_priv->ips.renderctx);
                 drm_gem_object_unreference(&dev_priv->ips.renderctx->base);
                 dev_priv->ips.renderctx = NULL;
         }
 
         if (dev_priv->ips.pwrctx) {
-                i915_gem_object_unpin(dev_priv->ips.pwrctx);
+                i915_gem_object_ggtt_unpin(dev_priv->ips.pwrctx);
                 drm_gem_object_unreference(&dev_priv->ips.pwrctx->base);
                 dev_priv->ips.pwrctx = NULL;
         }
@@ -3823,9 +3911,10 @@ static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
 
 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
 {
+        struct drm_device *dev = dev_priv->dev;
         unsigned long val;
 
-        if (dev_priv->info->gen != 5)
+        if (INTEL_INFO(dev)->gen != 5)
                 return 0;
 
         spin_lock_irq(&mchdev_lock);
@@ -3854,6 +3943,7 @@ unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
 
 static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
 {
+        struct drm_device *dev = dev_priv->dev;
         static const struct v_table {
                 u16 vd; /* in .1 mil */
                 u16 vm; /* in .1 mil */
@@ -3987,7 +4077,7 @@ static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
                 { 16000, 14875, },
                 { 16125, 15000, },
         };
-        if (dev_priv->info->is_mobile)
+        if (INTEL_INFO(dev)->is_mobile)
                 return v_table[pxvid].vm;
         else
                 return v_table[pxvid].vd;
@@ -4030,7 +4120,9 @@ static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
 
 void i915_update_gfx_val(struct drm_i915_private *dev_priv)
 {
-        if (dev_priv->info->gen != 5)
+        struct drm_device *dev = dev_priv->dev;
+
+        if (INTEL_INFO(dev)->gen != 5)
                 return;
 
         spin_lock_irq(&mchdev_lock);
@@ -4047,7 +4139,7 @@ static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
 
         assert_spin_locked(&mchdev_lock);
 
-        pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_delay * 4));
+        pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->rps.cur_freq * 4));
         pxvid = (pxvid >> 24) & 0x7f;
         ext_v = pvid_to_extvid(dev_priv, pxvid);
 
@@ -4079,9 +4171,10 @@ static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
 
 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
 {
+        struct drm_device *dev = dev_priv->dev;
         unsigned long val;
 
-        if (dev_priv->info->gen != 5)
+        if (INTEL_INFO(dev)->gen != 5)
                 return 0;
 
         spin_lock_irq(&mchdev_lock);
@@ -4270,6 +4363,7 @@ void intel_gpu_ips_teardown(void)
         i915_mch_dev = NULL;
         spin_unlock_irq(&mchdev_lock);
 }
+
 static void intel_init_emon(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -4341,6 +4435,18 @@ static void intel_init_emon(struct drm_device *dev)
         dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
 }
 
+void intel_init_gt_powersave(struct drm_device *dev)
+{
+        if (IS_VALLEYVIEW(dev))
+                valleyview_setup_pctx(dev);
+}
+
+void intel_cleanup_gt_powersave(struct drm_device *dev)
+{
+        if (IS_VALLEYVIEW(dev))
+                valleyview_cleanup_pctx(dev);
+}
+
 void intel_disable_gt_powersave(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -4395,8 +4501,6 @@ void intel_enable_gt_powersave(struct drm_device *dev)
                 ironlake_enable_rc6(dev);
                 intel_init_emon(dev);
         } else if (IS_GEN6(dev) || IS_GEN7(dev)) {
-                if (IS_VALLEYVIEW(dev))
-                        valleyview_setup_pctx(dev);
                 /*
                  * PCU communication is slow and this doesn't need to be
                  * done at any specific time, so do this out of our fast path
@@ -4587,6 +4691,17 @@ static void gen6_init_clock_gating(struct drm_device *dev)
                 I915_WRITE(GEN6_GT_MODE,
                            _MASKED_BIT_ENABLE(GEN6_TD_FOUR_ROW_DISPATCH_DISABLE));
 
+        /*
+         * BSpec recoomends 8x4 when MSAA is used,
+         * however in practice 16x4 seems fastest.
+         *
+         * Note that PS/WM thread counts depend on the WIZ hashing
+         * disable bit, which we don't touch here, but it's good
+         * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
+         */
+        I915_WRITE(GEN6_GT_MODE,
+                   GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
+
         ilk_init_lp_watermarks(dev);
 
         I915_WRITE(CACHE_MODE_0,
@@ -4607,17 +4722,24 @@ static void gen6_init_clock_gating(struct drm_device *dev)
          * According to the spec, bit 11 (RCCUNIT) must also be set,
          * but we didn't debug actual testcases to find it out.
          *
-         * Also apply WaDisableVDSUnitClockGating:snb and
-         * WaDisableRCPBUnitClockGating:snb.
+         * WaDisableRCCUnitClockGating:snb
+         * WaDisableRCPBUnitClockGating:snb
          */
         I915_WRITE(GEN6_UCGCTL2,
-                   GEN7_VDSUNIT_CLOCK_GATE_DISABLE |
                    GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
                    GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
 
-        /* Bspec says we need to always set all mask bits. */
-        I915_WRITE(_3D_CHICKEN3, (0xFFFF << 16) |
-                   _3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL);
+        /* WaStripsFansDisableFastClipPerformanceFix:snb */
+        I915_WRITE(_3D_CHICKEN3,
+                   _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_FASTCLIP_CULL));
+
+        /*
+         * Bspec says:
+         * "This bit must be set if 3DSTATE_CLIP clip mode is set to normal and
+         * 3DSTATE_SF number of SF output attributes is more than 16."
+         */
+        I915_WRITE(_3D_CHICKEN3,
+                   _MASKED_BIT_ENABLE(_3D_CHICKEN3_SF_DISABLE_PIPELINED_ATTR_FETCH));
 
         /*
          * According to the spec the following bits should be
@@ -4643,11 +4765,6 @@ static void gen6_init_clock_gating(struct drm_device *dev)
 
         g4x_disable_trickle_feed(dev);
 
-        /* The default value should be 0x200 according to docs, but the two
-         * platforms I checked have a 0 for this. (Maybe BIOS overrides?) */
-        I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_DISABLE(0xffff));
-        I915_WRITE(GEN6_GT_MODE, _MASKED_BIT_ENABLE(GEN6_GT_MODE_HI));
-
         cpt_init_clock_gating(dev);
 
         gen6_check_mch_setup(dev);
@@ -4657,14 +4774,17 @@ static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
 {
         uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
 
+        /*
+         * WaVSThreadDispatchOverride:ivb,vlv
+         *
+         * This actually overrides the dispatch
+         * mode for all thread types.
+         */
         reg &= ~GEN7_FF_SCHED_MASK;
         reg |= GEN7_FF_TS_SCHED_HW;
         reg |= GEN7_FF_VS_SCHED_HW;
         reg |= GEN7_FF_DS_SCHED_HW;
 
-        if (IS_HASWELL(dev_priv->dev))
-                reg &= ~GEN7_FF_VS_REF_CNT_FFME;
-
         I915_WRITE(GEN7_FF_THREAD_MODE, reg);
 }
 
@@ -4702,7 +4822,7 @@ static void lpt_suspend_hw(struct drm_device *dev)
 static void gen8_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        enum pipe i;
+        enum pipe pipe;
 
         I915_WRITE(WM3_LP_ILK, 0);
         I915_WRITE(WM2_LP_ILK, 0);
@@ -4711,8 +4831,19 @@ static void gen8_init_clock_gating(struct drm_device *dev)
         /* FIXME(BDW): Check all the w/a, some might only apply to
          * pre-production hw. */
 
-        WARN(!i915_preliminary_hw_support,
-             "GEN8_CENTROID_PIXEL_OPT_DIS not be needed for production\n");
+        /* WaDisablePartialInstShootdown:bdw */
+        I915_WRITE(GEN8_ROW_CHICKEN,
+                   _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE));
+
+        /* WaDisableThreadStallDopClockGating:bdw */
+        /* FIXME: Unclear whether we really need this on production bdw. */
+        I915_WRITE(GEN8_ROW_CHICKEN,
+                   _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
+
+        /*
+         * This GEN8_CENTROID_PIXEL_OPT_DIS W/A is only needed for
+         * pre-production hardware
+         */
         I915_WRITE(HALF_SLICE_CHICKEN3,
                    _MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS));
         I915_WRITE(HALF_SLICE_CHICKEN3,
@@ -4736,10 +4867,10 @@ static void gen8_init_clock_gating(struct drm_device *dev)
                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
 
         /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
-        for_each_pipe(i) {
-                I915_WRITE(CHICKEN_PIPESL_1(i),
-                           I915_READ(CHICKEN_PIPESL_1(i) |
-                                     DPRS_MASK_VBLANK_SRD));
+        for_each_pipe(pipe) {
+                I915_WRITE(CHICKEN_PIPESL_1(pipe),
+                           I915_READ(CHICKEN_PIPESL_1(pipe)) |
+                           BDW_DPRS_MASK_VBLANK_SRD);
         }
 
         /* Use Force Non-Coherent whenever executing a 3D context. This is a
@@ -4755,6 +4886,28 @@ static void gen8_init_clock_gating(struct drm_device *dev)
         I915_WRITE(GEN7_FF_THREAD_MODE,
                    I915_READ(GEN7_FF_THREAD_MODE) &
                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
+
+        /*
+         * BSpec recommends 8x4 when MSAA is used,
+         * however in practice 16x4 seems fastest.
+         *
+         * Note that PS/WM thread counts depend on the WIZ hashing
+         * disable bit, which we don't touch here, but it's good
+         * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
+         */
+        I915_WRITE(GEN7_GT_MODE,
+                   GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
+
+        I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
+                   _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
+
+        /* WaDisableSDEUnitClockGating:bdw */
+        I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
+                   GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
+
+        /* Wa4x4STCOptimizationDisable:bdw */
+        I915_WRITE(CACHE_MODE_1,
+                   _MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE));
 }
 
 static void haswell_init_clock_gating(struct drm_device *dev)
@@ -4763,21 +4916,6 @@ static void haswell_init_clock_gating(struct drm_device *dev)
 
         ilk_init_lp_watermarks(dev);
 
-        /* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
-         * This implements the WaDisableRCZUnitClockGating:hsw workaround.
-         */
-        I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
-
-        /* Apply the WaDisableRHWOOptimizationForRenderHang:hsw workaround. */
-        I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
-                   GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
-
-        /* WaApplyL3ControlAndL3ChickenMode:hsw */
-        I915_WRITE(GEN7_L3CNTLREG1,
-                        GEN7_WA_FOR_GEN7_L3_CONTROL);
-        I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
-                        GEN7_WA_L3_CHICKEN_MODE);
-
         /* L3 caching of data atomics doesn't work -- disable it. */
         I915_WRITE(HSW_SCRATCH1, HSW_SCRATCH1_L3_DATA_ATOMICS_DISABLE);
         I915_WRITE(HSW_ROW_CHICKEN3,
@@ -4789,12 +4927,28 @@ static void haswell_init_clock_gating(struct drm_device *dev)
                         GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 
         /* WaVSRefCountFullforceMissDisable:hsw */
-        gen7_setup_fixed_func_scheduler(dev_priv);
+        I915_WRITE(GEN7_FF_THREAD_MODE,
+                   I915_READ(GEN7_FF_THREAD_MODE) & ~GEN7_FF_VS_REF_CNT_FFME);
+
+        /* enable HiZ Raw Stall Optimization */
+        I915_WRITE(CACHE_MODE_0_GEN7,
+                   _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
 
         /* WaDisable4x2SubspanOptimization:hsw */
         I915_WRITE(CACHE_MODE_1,
                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
 
+        /*
+         * BSpec recommends 8x4 when MSAA is used,
+         * however in practice 16x4 seems fastest.
+         *
+         * Note that PS/WM thread counts depend on the WIZ hashing
+         * disable bit, which we don't touch here, but it's good
+         * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
+         */
+        I915_WRITE(GEN7_GT_MODE,
+                   GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
+
         /* WaSwitchSolVfFArbitrationPriority:hsw */
         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
 
@@ -4827,9 +4981,6 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
         if (IS_IVB_GT1(dev))
                 I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
                            _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
-        else
-                I915_WRITE(GEN7_HALF_SLICE_CHICKEN1_GT2,
-                           _MASKED_BIT_ENABLE(GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
 
         /* Apply the WaDisableRHWOOptimizationForRenderHang:ivb workaround. */
         I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
@@ -4843,31 +4994,24 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
         if (IS_IVB_GT1(dev))
                 I915_WRITE(GEN7_ROW_CHICKEN2,
                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
-        else
+        else {
+                /* must write both registers */
+                I915_WRITE(GEN7_ROW_CHICKEN2,
+                           _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
                 I915_WRITE(GEN7_ROW_CHICKEN2_GT2,
                            _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
-
+        }
 
         /* WaForceL3Serialization:ivb */
         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
 
-        /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
-         * gating disable must be set.  Failure to set it results in
-         * flickering pixels due to Z write ordering failures after
-         * some amount of runtime in the Mesa "fire" demo, and Unigine
-         * Sanctuary and Tropics, and apparently anything else with
-         * alpha test or pixel discard.
-         *
-         * According to the spec, bit 11 (RCCUNIT) must also be set,
-         * but we didn't debug actual testcases to find it out.
-         *
+        /*
          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
          * This implements the WaDisableRCZUnitClockGating:ivb workaround.
          */
         I915_WRITE(GEN6_UCGCTL2,
-                   GEN6_RCZUNIT_CLOCK_GATE_DISABLE |
-                   GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
+                   GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 
         /* This is required by WaCatErrorRejectionIssue:ivb */
         I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
@@ -4876,13 +5020,29 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
 
         g4x_disable_trickle_feed(dev);
 
-        /* WaVSRefCountFullforceMissDisable:ivb */
         gen7_setup_fixed_func_scheduler(dev_priv);
 
+        if (0) { /* causes HiZ corruption on ivb:gt1 */
+                /* enable HiZ Raw Stall Optimization */
+                I915_WRITE(CACHE_MODE_0_GEN7,
+                           _MASKED_BIT_DISABLE(HIZ_RAW_STALL_OPT_DISABLE));
+        }
+
         /* WaDisable4x2SubspanOptimization:ivb */
         I915_WRITE(CACHE_MODE_1,
                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
 
+        /*
+         * BSpec recommends 8x4 when MSAA is used,
+         * however in practice 16x4 seems fastest.
+         *
+         * Note that PS/WM thread counts depend on the WIZ hashing
+         * disable bit, which we don't touch here, but it's good
+         * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
+         */
+        I915_WRITE(GEN7_GT_MODE,
+                   GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
+
         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
         snpcr &= ~GEN6_MBC_SNPCR_MASK;
         snpcr |= GEN6_MBC_SNPCR_MED;
@@ -4904,13 +5064,11 @@ static void valleyview_init_clock_gating(struct drm_device *dev)
         mutex_unlock(&dev_priv->rps.hw_lock);
         switch ((val >> 6) & 3) {
         case 0:
-                dev_priv->mem_freq = 800;
-                break;
         case 1:
-                dev_priv->mem_freq = 1066;
+                dev_priv->mem_freq = 800;
                 break;
         case 2:
-                dev_priv->mem_freq = 1333;
+                dev_priv->mem_freq = 1066;
                 break;
         case 3:
                 dev_priv->mem_freq = 1333;
@@ -4929,19 +5087,12 @@ static void valleyview_init_clock_gating(struct drm_device *dev)
                    CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
                    CHICKEN3_DGMG_DONE_FIX_DISABLE);
 
+        /* WaPsdDispatchEnable:vlv */
         /* WaDisablePSDDualDispatchEnable:vlv */
         I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
                    _MASKED_BIT_ENABLE(GEN7_MAX_PS_THREAD_DEP |
                                       GEN7_PSD_SINGLE_PORT_DISPATCH_ENABLE));
 
-        /* Apply the WaDisableRHWOOptimizationForRenderHang:vlv workaround. */
-        I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
-                   GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
-
-        /* WaApplyL3ControlAndL3ChickenMode:vlv */
-        I915_WRITE(GEN7_L3CNTLREG1, I915_READ(GEN7_L3CNTLREG1) | GEN7_L3AGDIS);
-        I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
-
         /* WaForceL3Serialization:vlv */
         I915_WRITE(GEN7_L3SQCREG4, I915_READ(GEN7_L3SQCREG4) &
                    ~L3SQ_URB_READ_CAM_MATCH_DISABLE);
@@ -4955,51 +5106,39 @@ static void valleyview_init_clock_gating(struct drm_device *dev)
                    I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
                    GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
 
-        /* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
-         * gating disable must be set.  Failure to set it results in
-         * flickering pixels due to Z write ordering failures after
-         * some amount of runtime in the Mesa "fire" demo, and Unigine
-         * Sanctuary and Tropics, and apparently anything else with
-         * alpha test or pixel discard.
-         *
-         * According to the spec, bit 11 (RCCUNIT) must also be set,
-         * but we didn't debug actual testcases to find it out.
-         *
+        gen7_setup_fixed_func_scheduler(dev_priv);
+
+        /*
          * According to the spec, bit 13 (RCZUNIT) must be set on IVB.
          * This implements the WaDisableRCZUnitClockGating:vlv workaround.
-         *
-         * Also apply WaDisableVDSUnitClockGating:vlv and
-         * WaDisableRCPBUnitClockGating:vlv.
          */
         I915_WRITE(GEN6_UCGCTL2,
-                   GEN7_VDSUNIT_CLOCK_GATE_DISABLE |
-                   GEN7_TDLUNIT_CLOCK_GATE_DISABLE |
-                   GEN6_RCZUNIT_CLOCK_GATE_DISABLE |
-                   GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
-                   GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
+                   GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
 
+        /* WaDisableL3Bank2xClockGate:vlv */
         I915_WRITE(GEN7_UCGCTL4, GEN7_L3BANK2X_CLOCK_GATE_DISABLE);
 
         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
 
+        /*
+         * BSpec says this must be set, even though
+         * WaDisable4x2SubspanOptimization isn't listed for VLV.
+         */
         I915_WRITE(CACHE_MODE_1,
                    _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
 
         /*
+         * WaIncreaseL3CreditsForVLVB0:vlv
+         * This is the hardware default actually.
+         */
+        I915_WRITE(GEN7_L3SQCREG1, VLV_B0_WA_L3SQCREG1_VALUE);
+
+        /*
          * WaDisableVLVClockGating_VBIIssue:vlv
          * Disable clock gating on th GCFG unit to prevent a delay
          * in the reporting of vblank events.
          */
-        I915_WRITE(VLV_GUNIT_CLOCK_GATE, 0xffffffff);
-
-        /* Conservative clock gating settings for now */
-        I915_WRITE(0x9400, 0xffffffff);
-        I915_WRITE(0x9404, 0xffffffff);
-        I915_WRITE(0x9408, 0xffffffff);
-        I915_WRITE(0x940c, 0xffffffff);
-        I915_WRITE(0x9410, 0xffffffff);
-        I915_WRITE(0x9414, 0xffffffff);
-        I915_WRITE(0x9418, 0xffffffff);
+        I915_WRITE(VLV_GUNIT_CLOCK_GATE, GCFG_DIS);
 }
 
 static void g4x_init_clock_gating(struct drm_device *dev)
@@ -5114,19 +5253,16 @@ void intel_suspend_hw(struct drm_device *dev)
  * enable it, so check if it's enabled and also check if we've requested it to
  * be enabled.
  */
-static bool hsw_power_well_enabled(struct drm_device *dev,
+static bool hsw_power_well_enabled(struct drm_i915_private *dev_priv,
                                    struct i915_power_well *power_well)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         return I915_READ(HSW_PWR_WELL_DRIVER) ==
                      (HSW_PWR_WELL_ENABLE_REQUEST | HSW_PWR_WELL_STATE_ENABLED);
 }
 
-bool intel_display_power_enabled_sw(struct drm_device *dev,
+bool intel_display_power_enabled_sw(struct drm_i915_private *dev_priv,
                                     enum intel_display_power_domain domain)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct i915_power_domains *power_domains;
 
         power_domains = &dev_priv->power_domains;
@@ -5134,15 +5270,17 @@ bool intel_display_power_enabled_sw(struct drm_device *dev,
         return power_domains->domain_use_count[domain];
 }
 
-bool intel_display_power_enabled(struct drm_device *dev,
+bool intel_display_power_enabled(struct drm_i915_private *dev_priv,
                                  enum intel_display_power_domain domain)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct i915_power_domains *power_domains;
         struct i915_power_well *power_well;
         bool is_enabled;
         int i;
 
+        if (dev_priv->pm.suspended)
+                return false;
+
         power_domains = &dev_priv->power_domains;
 
         is_enabled = true;
@@ -5152,7 +5290,7 @@ bool intel_display_power_enabled(struct drm_device *dev,
                 if (power_well->always_on)
                         continue;
 
-                if (!power_well->is_enabled(dev, power_well)) {
+                if (!power_well->ops->is_enabled(dev_priv, power_well)) {
                         is_enabled = false;
                         break;
                 }
@@ -5162,6 +5300,12 @@ bool intel_display_power_enabled(struct drm_device *dev,
         return is_enabled;
 }
 
+/*
+ * Starting with Haswell, we have a "Power Down Well" that can be turned off
+ * when not needed anymore. We have 4 registers that can request the power well
+ * to be enabled, and it will only be disabled if none of the registers is
+ * requesting it to be enabled.
+ */
 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
 {
         struct drm_device *dev = dev_priv->dev;
@@ -5198,10 +5342,17 @@ static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
         }
 }
 
+static void reset_vblank_counter(struct drm_device *dev, enum pipe pipe)
+{
+        assert_spin_locked(&dev->vbl_lock);
+
+        dev->vblank[pipe].last = 0;
+}
+
 static void hsw_power_well_post_disable(struct drm_i915_private *dev_priv)
 {
         struct drm_device *dev = dev_priv->dev;
-        enum pipe p;
+        enum pipe pipe;
         unsigned long irqflags;
 
         /*
@@ -5212,21 +5363,18 @@ static void hsw_power_well_post_disable(struct drm_i915_private *dev_priv)
          * FIXME: Should we do this in general in drm_vblank_post_modeset?
          */
         spin_lock_irqsave(&dev->vbl_lock, irqflags);
-        for_each_pipe(p)
-                if (p != PIPE_A)
-                        dev->vblank[p].last = 0;
+        for_each_pipe(pipe)
+                if (pipe != PIPE_A)
+                        reset_vblank_counter(dev, pipe);
         spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
 }
 
-static void hsw_set_power_well(struct drm_device *dev,
+static void hsw_set_power_well(struct drm_i915_private *dev_priv,
                                struct i915_power_well *power_well, bool enable)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         bool is_enabled, enable_requested;
         uint32_t tmp;
 
-        WARN_ON(dev_priv->pc8.enabled);
-
         tmp = I915_READ(HSW_PWR_WELL_DRIVER);
         is_enabled = tmp & HSW_PWR_WELL_STATE_ENABLED;
         enable_requested = tmp & HSW_PWR_WELL_ENABLE_REQUEST;
@@ -5255,55 +5403,229 @@ static void hsw_set_power_well(struct drm_device *dev,
         }
 }
 
-static void __intel_power_well_get(struct drm_device *dev,
+static void hsw_power_well_sync_hw(struct drm_i915_private *dev_priv,
                                    struct i915_power_well *power_well)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        hsw_set_power_well(dev_priv, power_well, power_well->count > 0);
 
-        if (!power_well->count++ && power_well->set) {
-                hsw_disable_package_c8(dev_priv);
-                power_well->set(dev, power_well, true);
-        }
+        /*
+         * We're taking over the BIOS, so clear any requests made by it since
+         * the driver is in charge now.
+         */
+        if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST)
+                I915_WRITE(HSW_PWR_WELL_BIOS, 0);
+}
+
+static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
+                                  struct i915_power_well *power_well)
+{
+        hsw_set_power_well(dev_priv, power_well, true);
 }
 
-static void __intel_power_well_put(struct drm_device *dev,
+static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
                                    struct i915_power_well *power_well)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        hsw_set_power_well(dev_priv, power_well, false);
+}
+
+static void i9xx_always_on_power_well_noop(struct drm_i915_private *dev_priv,
+                                           struct i915_power_well *power_well)
+{
+}
+
+static bool i9xx_always_on_power_well_enabled(struct drm_i915_private *dev_priv,
+                                             struct i915_power_well *power_well)
+{
+        return true;
+}
+
+static void vlv_set_power_well(struct drm_i915_private *dev_priv,
+                               struct i915_power_well *power_well, bool enable)
+{
+        enum punit_power_well power_well_id = power_well->data;
+        u32 mask;
+        u32 state;
+        u32 ctrl;
+
+        mask = PUNIT_PWRGT_MASK(power_well_id);
+        state = enable ? PUNIT_PWRGT_PWR_ON(power_well_id) :
+                         PUNIT_PWRGT_PWR_GATE(power_well_id);
+
+        mutex_lock(&dev_priv->rps.hw_lock);
+
+#define COND \
+        ((vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask) == state)
+
+        if (COND)
+                goto out;
+
+        ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL);
+        ctrl &= ~mask;
+        ctrl |= state;
+        vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, ctrl);
+
+        if (wait_for(COND, 100))
+                DRM_ERROR("timout setting power well state %08x (%08x)\n",
+                          state,
+                          vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL));
+
+#undef COND
+
+out:
+        mutex_unlock(&dev_priv->rps.hw_lock);
+}
+
+static void vlv_power_well_sync_hw(struct drm_i915_private *dev_priv,
+                                   struct i915_power_well *power_well)
+{
+        vlv_set_power_well(dev_priv, power_well, power_well->count > 0);
+}
+
+static void vlv_power_well_enable(struct drm_i915_private *dev_priv,
+                                  struct i915_power_well *power_well)
+{
+        vlv_set_power_well(dev_priv, power_well, true);
+}
+
+static void vlv_power_well_disable(struct drm_i915_private *dev_priv,
+                                   struct i915_power_well *power_well)
+{
+        vlv_set_power_well(dev_priv, power_well, false);
+}
+
+static bool vlv_power_well_enabled(struct drm_i915_private *dev_priv,
+                                   struct i915_power_well *power_well)
+{
+        int power_well_id = power_well->data;
+        bool enabled = false;
+        u32 mask;
+        u32 state;
+        u32 ctrl;
+
+        mask = PUNIT_PWRGT_MASK(power_well_id);
+        ctrl = PUNIT_PWRGT_PWR_ON(power_well_id);
+
+        mutex_lock(&dev_priv->rps.hw_lock);
+
+        state = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS) & mask;
+        /*
+         * We only ever set the power-on and power-gate states, anything
+         * else is unexpected.
+         */
+        WARN_ON(state != PUNIT_PWRGT_PWR_ON(power_well_id) &&
+                state != PUNIT_PWRGT_PWR_GATE(power_well_id));
+        if (state == ctrl)
+                enabled = true;
+
+        /*
+         * A transient state at this point would mean some unexpected party
+         * is poking at the power controls too.
+         */
+        ctrl = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_CTRL) & mask;
+        WARN_ON(ctrl != state);
+
+        mutex_unlock(&dev_priv->rps.hw_lock);
+
+        return enabled;
+}
 
-        WARN_ON(!power_well->count);
+static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
+                                          struct i915_power_well *power_well)
+{
+        WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D);
+
+        vlv_set_power_well(dev_priv, power_well, true);
+
+        spin_lock_irq(&dev_priv->irq_lock);
+        valleyview_enable_display_irqs(dev_priv);
+        spin_unlock_irq(&dev_priv->irq_lock);
+
+        /*
+         * During driver initialization we need to defer enabling hotplug
+         * processing until fbdev is set up.
+         */
+        if (dev_priv->enable_hotplug_processing)
+                intel_hpd_init(dev_priv->dev);
+
+        i915_redisable_vga_power_on(dev_priv->dev);
+}
+
+static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
+                                           struct i915_power_well *power_well)
+{
+        struct drm_device *dev = dev_priv->dev;
+        enum pipe pipe;
+
+        WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DISP2D);
+
+        spin_lock_irq(&dev_priv->irq_lock);
+        for_each_pipe(pipe)
+                __intel_set_cpu_fifo_underrun_reporting(dev, pipe, false);
+
+        valleyview_disable_display_irqs(dev_priv);
+        spin_unlock_irq(&dev_priv->irq_lock);
+
+        spin_lock_irq(&dev->vbl_lock);
+        for_each_pipe(pipe)
+                reset_vblank_counter(dev, pipe);
+        spin_unlock_irq(&dev->vbl_lock);
+
+        vlv_set_power_well(dev_priv, power_well, false);
+}
 
-        if (!--power_well->count && power_well->set &&
-            i915_disable_power_well) {
-                power_well->set(dev, power_well, false);
-                hsw_enable_package_c8(dev_priv);
+static void check_power_well_state(struct drm_i915_private *dev_priv,
+                                   struct i915_power_well *power_well)
+{
+        bool enabled = power_well->ops->is_enabled(dev_priv, power_well);
+
+        if (power_well->always_on || !i915.disable_power_well) {
+                if (!enabled)
+                        goto mismatch;
+
+                return;
         }
+
+        if (enabled != (power_well->count > 0))
+                goto mismatch;
+
+        return;
+
+mismatch:
+        WARN(1, "state mismatch for '%s' (always_on %d hw state %d use-count %d disable_power_well %d\n",
+                  power_well->name, power_well->always_on, enabled,
+                  power_well->count, i915.disable_power_well);
 }
 
-void intel_display_power_get(struct drm_device *dev,
+void intel_display_power_get(struct drm_i915_private *dev_priv,
                              enum intel_display_power_domain domain)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct i915_power_domains *power_domains;
         struct i915_power_well *power_well;
         int i;
 
+        intel_runtime_pm_get(dev_priv);
+
         power_domains = &dev_priv->power_domains;
 
         mutex_lock(&power_domains->lock);
 
-        for_each_power_well(i, power_well, BIT(domain), power_domains)
-                __intel_power_well_get(dev, power_well);
+        for_each_power_well(i, power_well, BIT(domain), power_domains) {
+                if (!power_well->count++) {
+                        DRM_DEBUG_KMS("enabling %s\n", power_well->name);
+                        power_well->ops->enable(dev_priv, power_well);
+                }
+
+                check_power_well_state(dev_priv, power_well);
+        }
 
         power_domains->domain_use_count[domain]++;
 
         mutex_unlock(&power_domains->lock);
 }
 
-void intel_display_power_put(struct drm_device *dev,
+void intel_display_power_put(struct drm_i915_private *dev_priv,
                              enum intel_display_power_domain domain)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct i915_power_domains *power_domains;
         struct i915_power_well *power_well;
         int i;
@@ -5315,10 +5637,20 @@ void intel_display_power_put(struct drm_device *dev,
         WARN_ON(!power_domains->domain_use_count[domain]);
         power_domains->domain_use_count[domain]--;
 
-        for_each_power_well_rev(i, power_well, BIT(domain), power_domains)
-                __intel_power_well_put(dev, power_well);
+        for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
+                WARN_ON(!power_well->count);
+
+                if (!--power_well->count && i915.disable_power_well) {
+                        DRM_DEBUG_KMS("disabling %s\n", power_well->name);
+                        power_well->ops->disable(dev_priv, power_well);
+                }
+
+                check_power_well_state(dev_priv, power_well);
+        }
 
         mutex_unlock(&power_domains->lock);
+
+        intel_runtime_pm_put(dev_priv);
 }
 
 static struct i915_power_domains *hsw_pwr;
@@ -5333,7 +5665,7 @@ void i915_request_power_well(void)
 
         dev_priv = container_of(hsw_pwr, struct drm_i915_private,
                                 power_domains);
-        intel_display_power_get(dev_priv->dev, POWER_DOMAIN_AUDIO);
+        intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO);
 }
 EXPORT_SYMBOL_GPL(i915_request_power_well);
 
@@ -5347,29 +5679,99 @@ void i915_release_power_well(void)
 
         dev_priv = container_of(hsw_pwr, struct drm_i915_private,
                                 power_domains);
-        intel_display_power_put(dev_priv->dev, POWER_DOMAIN_AUDIO);
+        intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO);
 }
 EXPORT_SYMBOL_GPL(i915_release_power_well);
 
+#define POWER_DOMAIN_MASK (BIT(POWER_DOMAIN_NUM) - 1)
+
+#define HSW_ALWAYS_ON_POWER_DOMAINS (                   \
+        BIT(POWER_DOMAIN_PIPE_A) |                      \
+        BIT(POWER_DOMAIN_TRANSCODER_EDP) |              \
+        BIT(POWER_DOMAIN_PORT_DDI_A_2_LANES) |          \
+        BIT(POWER_DOMAIN_PORT_DDI_A_4_LANES) |          \
+        BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |          \
+        BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |          \
+        BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |          \
+        BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |          \
+        BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |          \
+        BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |          \
+        BIT(POWER_DOMAIN_PORT_CRT) |                    \
+        BIT(POWER_DOMAIN_INIT))
+#define HSW_DISPLAY_POWER_DOMAINS (                             \
+        (POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS) |    \
+        BIT(POWER_DOMAIN_INIT))
+
+#define BDW_ALWAYS_ON_POWER_DOMAINS (                   \
+        HSW_ALWAYS_ON_POWER_DOMAINS |                   \
+        BIT(POWER_DOMAIN_PIPE_A_PANEL_FITTER))
+#define BDW_DISPLAY_POWER_DOMAINS (                             \
+        (POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS) |    \
+        BIT(POWER_DOMAIN_INIT))
+
+#define VLV_ALWAYS_ON_POWER_DOMAINS     BIT(POWER_DOMAIN_INIT)
+#define VLV_DISPLAY_POWER_DOMAINS       POWER_DOMAIN_MASK
+
+#define VLV_DPIO_CMN_BC_POWER_DOMAINS (         \
+        BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |  \
+        BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
+        BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |  \
+        BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
+        BIT(POWER_DOMAIN_PORT_CRT) |            \
+        BIT(POWER_DOMAIN_INIT))
+
+#define VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS (  \
+        BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |  \
+        BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
+        BIT(POWER_DOMAIN_INIT))
+
+#define VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS (  \
+        BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
+        BIT(POWER_DOMAIN_INIT))
+
+#define VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS (  \
+        BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |  \
+        BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
+        BIT(POWER_DOMAIN_INIT))
+
+#define VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS (  \
+        BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
+        BIT(POWER_DOMAIN_INIT))
+
+static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
+        .sync_hw = i9xx_always_on_power_well_noop,
+        .enable = i9xx_always_on_power_well_noop,
+        .disable = i9xx_always_on_power_well_noop,
+        .is_enabled = i9xx_always_on_power_well_enabled,
+};
+
 static struct i915_power_well i9xx_always_on_power_well[] = {
         {
                 .name = "always-on",
                 .always_on = 1,
                 .domains = POWER_DOMAIN_MASK,
+                .ops = &i9xx_always_on_power_well_ops,
         },
 };
 
+static const struct i915_power_well_ops hsw_power_well_ops = {
+        .sync_hw = hsw_power_well_sync_hw,
+        .enable = hsw_power_well_enable,
+        .disable = hsw_power_well_disable,
+        .is_enabled = hsw_power_well_enabled,
+};
+
 static struct i915_power_well hsw_power_wells[] = {
         {
                 .name = "always-on",
                 .always_on = 1,
                 .domains = HSW_ALWAYS_ON_POWER_DOMAINS,
+                .ops = &i9xx_always_on_power_well_ops,
         },
         {
                 .name = "display",
-                .domains = POWER_DOMAIN_MASK & ~HSW_ALWAYS_ON_POWER_DOMAINS,
-                .is_enabled = hsw_power_well_enabled,
-                .set = hsw_set_power_well,
+                .domains = HSW_DISPLAY_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
         },
 };
 
@@ -5378,12 +5780,83 @@ static struct i915_power_well bdw_power_wells[] = {
                 .name = "always-on",
                 .always_on = 1,
                 .domains = BDW_ALWAYS_ON_POWER_DOMAINS,
+                .ops = &i9xx_always_on_power_well_ops,
         },
         {
                 .name = "display",
-                .domains = POWER_DOMAIN_MASK & ~BDW_ALWAYS_ON_POWER_DOMAINS,
-                .is_enabled = hsw_power_well_enabled,
-                .set = hsw_set_power_well,
+                .domains = BDW_DISPLAY_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+        },
+};
+
+static const struct i915_power_well_ops vlv_display_power_well_ops = {
+        .sync_hw = vlv_power_well_sync_hw,
+        .enable = vlv_display_power_well_enable,
+        .disable = vlv_display_power_well_disable,
+        .is_enabled = vlv_power_well_enabled,
+};
+
+static const struct i915_power_well_ops vlv_dpio_power_well_ops = {
+        .sync_hw = vlv_power_well_sync_hw,
+        .enable = vlv_power_well_enable,
+        .disable = vlv_power_well_disable,
+        .is_enabled = vlv_power_well_enabled,
+};
+
+static struct i915_power_well vlv_power_wells[] = {
+        {
+                .name = "always-on",
+                .always_on = 1,
+                .domains = VLV_ALWAYS_ON_POWER_DOMAINS,
+                .ops = &i9xx_always_on_power_well_ops,
+        },
+        {
+                .name = "display",
+                .domains = VLV_DISPLAY_POWER_DOMAINS,
+                .data = PUNIT_POWER_WELL_DISP2D,
+                .ops = &vlv_display_power_well_ops,
+        },
+        {
+                .name = "dpio-common",
+                .domains = VLV_DPIO_CMN_BC_POWER_DOMAINS,
+                .data = PUNIT_POWER_WELL_DPIO_CMN_BC,
+                .ops = &vlv_dpio_power_well_ops,
+        },
+        {
+                .name = "dpio-tx-b-01",
+                .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
+                           VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
+                           VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
+                           VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
+                .ops = &vlv_dpio_power_well_ops,
+                .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
+        },
+        {
+                .name = "dpio-tx-b-23",
+                .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
+                           VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
+                           VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
+                           VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
+                .ops = &vlv_dpio_power_well_ops,
+                .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
+        },
+        {
+                .name = "dpio-tx-c-01",
+                .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
+                           VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
+                           VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
+                           VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
+                .ops = &vlv_dpio_power_well_ops,
+                .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
+        },
+        {
+                .name = "dpio-tx-c-23",
+                .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
+                           VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS |
+                           VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
+                           VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
+                .ops = &vlv_dpio_power_well_ops,
+                .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
         },
 };
 
@@ -5392,9 +5865,8 @@ static struct i915_power_well bdw_power_wells[] = {
         (power_domains)->power_well_count = ARRAY_SIZE(__power_wells);  \
 })
 
-int intel_power_domains_init(struct drm_device *dev)
+int intel_power_domains_init(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct i915_power_domains *power_domains = &dev_priv->power_domains;
 
         mutex_init(&power_domains->lock);
@@ -5403,12 +5875,14 @@ int intel_power_domains_init(struct drm_device *dev)
          * The enabling order will be from lower to higher indexed wells,
          * the disabling order is reversed.
          */
-        if (IS_HASWELL(dev)) {
+        if (IS_HASWELL(dev_priv->dev)) {
                 set_power_wells(power_domains, hsw_power_wells);
                 hsw_pwr = power_domains;
-        } else if (IS_BROADWELL(dev)) {
+        } else if (IS_BROADWELL(dev_priv->dev)) {
                 set_power_wells(power_domains, bdw_power_wells);
                 hsw_pwr = power_domains;
+        } else if (IS_VALLEYVIEW(dev_priv->dev)) {
+                set_power_wells(power_domains, vlv_power_wells);
         } else {
                 set_power_wells(power_domains, i9xx_always_on_power_well);
         }
@@ -5416,58 +5890,38 @@ int intel_power_domains_init(struct drm_device *dev)
         return 0;
 }
 
-void intel_power_domains_remove(struct drm_device *dev)
+void intel_power_domains_remove(struct drm_i915_private *dev_priv)
 {
         hsw_pwr = NULL;
 }
 
-static void intel_power_domains_resume(struct drm_device *dev)
+static void intel_power_domains_resume(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct i915_power_domains *power_domains = &dev_priv->power_domains;
         struct i915_power_well *power_well;
         int i;
 
         mutex_lock(&power_domains->lock);
-        for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains) {
-                if (power_well->set)
-                        power_well->set(dev, power_well, power_well->count > 0);
-        }
+        for_each_power_well(i, power_well, POWER_DOMAIN_MASK, power_domains)
+                power_well->ops->sync_hw(dev_priv, power_well);
         mutex_unlock(&power_domains->lock);
 }
 
-/*
- * Starting with Haswell, we have a "Power Down Well" that can be turned off
- * when not needed anymore. We have 4 registers that can request the power well
- * to be enabled, and it will only be disabled if none of the registers is
- * requesting it to be enabled.
- */
-void intel_power_domains_init_hw(struct drm_device *dev)
+void intel_power_domains_init_hw(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         /* For now, we need the power well to be always enabled. */
-        intel_display_set_init_power(dev, true);
-        intel_power_domains_resume(dev);
-
-        if (!(IS_HASWELL(dev) || IS_BROADWELL(dev)))
-                return;
-
-        /* We're taking over the BIOS, so clear any requests made by it since
-         * the driver is in charge now. */
-        if (I915_READ(HSW_PWR_WELL_BIOS) & HSW_PWR_WELL_ENABLE_REQUEST)
-                I915_WRITE(HSW_PWR_WELL_BIOS, 0);
+        intel_display_set_init_power(dev_priv, true);
+        intel_power_domains_resume(dev_priv);
 }
 
-/* Disables PC8 so we can use the GMBUS and DP AUX interrupts. */
 void intel_aux_display_runtime_get(struct drm_i915_private *dev_priv)
 {
-        hsw_disable_package_c8(dev_priv);
+        intel_runtime_pm_get(dev_priv);
 }
 
 void intel_aux_display_runtime_put(struct drm_i915_private *dev_priv)
 {
-        hsw_enable_package_c8(dev_priv);
+        intel_runtime_pm_put(dev_priv);
 }
 
 void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
@@ -5499,8 +5953,6 @@ void intel_init_runtime_pm(struct drm_i915_private *dev_priv)
         struct drm_device *dev = dev_priv->dev;
         struct device *device = &dev->pdev->dev;
 
-        dev_priv->pm.suspended = false;
-
         if (!HAS_RUNTIME_PM(dev))
                 return;
 
@@ -5509,6 +5961,8 @@ void intel_init_runtime_pm(struct drm_i915_private *dev_priv)
         pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */
         pm_runtime_mark_last_busy(device);
         pm_runtime_use_autosuspend(device);
+
+        pm_runtime_put_autosuspend(device);
 }
 
 void intel_fini_runtime_pm(struct drm_i915_private *dev_priv)
@@ -5560,7 +6014,7 @@ void intel_init_pm(struct drm_device *dev)
 
         /* For FIFO watermark updates */
         if (HAS_PCH_SPLIT(dev)) {
-                intel_setup_wm_latency(dev);
+                ilk_setup_wm_latency(dev);
 
                 if ((IS_GEN5(dev) && dev_priv->wm.pri_latency[1] &&
                      dev_priv->wm.spr_latency[1] && dev_priv->wm.cur_latency[1]) ||
@@ -5731,13 +6185,9 @@ void intel_pm_setup(struct drm_device *dev)
 
         mutex_init(&dev_priv->rps.hw_lock);
 
-        mutex_init(&dev_priv->pc8.lock);
-        dev_priv->pc8.requirements_met = false;
-        dev_priv->pc8.gpu_idle = false;
-        dev_priv->pc8.irqs_disabled = false;
-        dev_priv->pc8.enabled = false;
-        dev_priv->pc8.disable_count = 2; /* requirements_met + gpu_idle */
-        INIT_DELAYED_WORK(&dev_priv->pc8.enable_work, hsw_enable_pc8_work);
         INIT_DELAYED_WORK(&dev_priv->rps.delayed_resume_work,
                           intel_gen6_powersave_work);
+
+        dev_priv->pm.suspended = false;
+        dev_priv->pm.irqs_disabled = false;
 }
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 31b36c5ac894..6bc68bdcf433 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -406,17 +406,24 @@ gen8_render_ring_flush(struct intel_ring_buffer *ring,
 static void ring_write_tail(struct intel_ring_buffer *ring,
                             u32 value)
 {
-        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
         I915_WRITE_TAIL(ring, value);
 }
 
-u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
+u64 intel_ring_get_active_head(struct intel_ring_buffer *ring)
 {
-        drm_i915_private_t *dev_priv = ring->dev->dev_private;
-        u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
-                        RING_ACTHD(ring->mmio_base) : ACTHD;
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
+        u64 acthd;
+
+        if (INTEL_INFO(ring->dev)->gen >= 8)
+                acthd = I915_READ64_2x32(RING_ACTHD(ring->mmio_base),
+                                         RING_ACTHD_UDW(ring->mmio_base));
+        else if (INTEL_INFO(ring->dev)->gen >= 4)
+                acthd = I915_READ(RING_ACTHD(ring->mmio_base));
+        else
+                acthd = I915_READ(ACTHD);
 
-        return I915_READ(acthd_reg);
+        return acthd;
 }
 
 static void ring_setup_phys_status_page(struct intel_ring_buffer *ring)
@@ -433,22 +440,24 @@ static void ring_setup_phys_status_page(struct intel_ring_buffer *ring)
 static int init_ring_common(struct intel_ring_buffer *ring)
 {
         struct drm_device *dev = ring->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_object *obj = ring->obj;
         int ret = 0;
         u32 head;
 
         gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
 
-        if (I915_NEED_GFX_HWS(dev))
-                intel_ring_setup_status_page(ring);
-        else
-                ring_setup_phys_status_page(ring);
-
         /* Stop the ring if it's running. */
         I915_WRITE_CTL(ring, 0);
         I915_WRITE_HEAD(ring, 0);
         ring->write_tail(ring, 0);
+        if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000))
+                DRM_ERROR("%s :timed out trying to stop ring\n", ring->name);
+
+        if (I915_NEED_GFX_HWS(dev))
+                intel_ring_setup_status_page(ring);
+        else
+                ring_setup_phys_status_page(ring);
 
         head = I915_READ_HEAD(ring) & HEAD_ADDR;
 
@@ -531,9 +540,11 @@ init_pipe_control(struct intel_ring_buffer *ring)
                 goto err;
         }
 
-        i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
+        ret = i915_gem_object_set_cache_level(ring->scratch.obj, I915_CACHE_LLC);
+        if (ret)
+                goto err_unref;
 
-        ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, true, false);
+        ret = i915_gem_obj_ggtt_pin(ring->scratch.obj, 4096, 0);
         if (ret)
                 goto err_unref;
 
@@ -549,7 +560,7 @@ init_pipe_control(struct intel_ring_buffer *ring)
         return 0;
 
 err_unpin:
-        i915_gem_object_unpin(ring->scratch.obj);
+        i915_gem_object_ggtt_unpin(ring->scratch.obj);
 err_unref:
         drm_gem_object_unreference(&ring->scratch.obj->base);
 err:
@@ -562,14 +573,15 @@ static int init_render_ring(struct intel_ring_buffer *ring)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int ret = init_ring_common(ring);
 
-        if (INTEL_INFO(dev)->gen > 3)
+        /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
+        if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7)
                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
 
         /* We need to disable the AsyncFlip performance optimisations in order
          * to use MI_WAIT_FOR_EVENT within the CS. It should already be
          * programmed to '1' on all products.
          *
-         * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
+         * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw
          */
         if (INTEL_INFO(dev)->gen >= 6)
                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
@@ -625,7 +637,7 @@ static void render_ring_cleanup(struct intel_ring_buffer *ring)
 
         if (INTEL_INFO(dev)->gen >= 5) {
                 kunmap(sg_page(ring->scratch.obj->pages->sgl));
-                i915_gem_object_unpin(ring->scratch.obj);
+                i915_gem_object_ggtt_unpin(ring->scratch.obj);
         }
 
         drm_gem_object_unreference(&ring->scratch.obj->base);
@@ -809,8 +821,11 @@ gen6_ring_get_seqno(struct intel_ring_buffer *ring, bool lazy_coherency)
         /* Workaround to force correct ordering between irq and seqno writes on
          * ivb (and maybe also on snb) by reading from a CS register (like
          * ACTHD) before reading the status page. */
-        if (!lazy_coherency)
-                intel_ring_get_active_head(ring);
+        if (!lazy_coherency) {
+                struct drm_i915_private *dev_priv = ring->dev->dev_private;
+                POSTING_READ(RING_ACTHD(ring->mmio_base));
+        }
+
         return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
 }
 
@@ -842,7 +857,7 @@ static bool
 gen5_ring_get_irq(struct intel_ring_buffer *ring)
 {
         struct drm_device *dev = ring->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long flags;
 
         if (!dev->irq_enabled)
@@ -860,7 +875,7 @@ static void
 gen5_ring_put_irq(struct intel_ring_buffer *ring)
 {
         struct drm_device *dev = ring->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long flags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
@@ -873,7 +888,7 @@ static bool
 i9xx_ring_get_irq(struct intel_ring_buffer *ring)
 {
         struct drm_device *dev = ring->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long flags;
 
         if (!dev->irq_enabled)
@@ -894,7 +909,7 @@ static void
 i9xx_ring_put_irq(struct intel_ring_buffer *ring)
 {
         struct drm_device *dev = ring->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long flags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
@@ -910,7 +925,7 @@ static bool
 i8xx_ring_get_irq(struct intel_ring_buffer *ring)
 {
         struct drm_device *dev = ring->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long flags;
 
         if (!dev->irq_enabled)
@@ -931,7 +946,7 @@ static void
 i8xx_ring_put_irq(struct intel_ring_buffer *ring)
 {
         struct drm_device *dev = ring->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long flags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
@@ -946,7 +961,7 @@ i8xx_ring_put_irq(struct intel_ring_buffer *ring)
 void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
 {
         struct drm_device *dev = ring->dev;
-        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
         u32 mmio = 0;
 
         /* The ring status page addresses are no longer next to the rest of
@@ -977,9 +992,19 @@ void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
         I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
         POSTING_READ(mmio);
 
-        /* Flush the TLB for this page */
-        if (INTEL_INFO(dev)->gen >= 6) {
+        /*
+         * Flush the TLB for this page
+         *
+         * FIXME: These two bits have disappeared on gen8, so a question
+         * arises: do we still need this and if so how should we go about
+         * invalidating the TLB?
+         */
+        if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
                 u32 reg = RING_INSTPM(ring->mmio_base);
+
+                /* ring should be idle before issuing a sync flush*/
+                WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
+
                 I915_WRITE(reg,
                            _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
                                               INSTPM_SYNC_FLUSH));
@@ -1029,7 +1054,7 @@ static bool
 gen6_ring_get_irq(struct intel_ring_buffer *ring)
 {
         struct drm_device *dev = ring->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long flags;
 
         if (!dev->irq_enabled)
@@ -1054,7 +1079,7 @@ static void
 gen6_ring_put_irq(struct intel_ring_buffer *ring)
 {
         struct drm_device *dev = ring->dev;
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long flags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
@@ -1253,7 +1278,7 @@ static void cleanup_status_page(struct intel_ring_buffer *ring)
                 return;
 
         kunmap(sg_page(obj->pages->sgl));
-        i915_gem_object_unpin(obj);
+        i915_gem_object_ggtt_unpin(obj);
         drm_gem_object_unreference(&obj->base);
         ring->status_page.obj = NULL;
 }
@@ -1271,12 +1296,13 @@ static int init_status_page(struct intel_ring_buffer *ring)
                 goto err;
         }
 
-        i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
+        ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
+        if (ret)
+                goto err_unref;
 
-        ret = i915_gem_obj_ggtt_pin(obj, 4096, true, false);
-        if (ret != 0) {
+        ret = i915_gem_obj_ggtt_pin(obj, 4096, 0);
+        if (ret)
                 goto err_unref;
-        }
 
         ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(obj);
         ring->status_page.page_addr = kmap(sg_page(obj->pages->sgl));
@@ -1293,7 +1319,7 @@ static int init_status_page(struct intel_ring_buffer *ring)
         return 0;
 
 err_unpin:
-        i915_gem_object_unpin(obj);
+        i915_gem_object_ggtt_unpin(obj);
 err_unref:
         drm_gem_object_unreference(&obj->base);
 err:
@@ -1356,7 +1382,7 @@ static int intel_init_ring_buffer(struct drm_device *dev,
 
         ring->obj = obj;
 
-        ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, true, false);
+        ret = i915_gem_obj_ggtt_pin(obj, PAGE_SIZE, PIN_MAPPABLE);
         if (ret)
                 goto err_unref;
 
@@ -1385,12 +1411,14 @@ static int intel_init_ring_buffer(struct drm_device *dev,
         if (IS_I830(ring->dev) || IS_845G(ring->dev))
                 ring->effective_size -= 128;
 
+        i915_cmd_parser_init_ring(ring);
+
         return 0;
 
 err_unmap:
         iounmap(ring->virtual_start);
 err_unpin:
-        i915_gem_object_unpin(obj);
+        i915_gem_object_ggtt_unpin(obj);
 err_unref:
         drm_gem_object_unreference(&obj->base);
         ring->obj = NULL;
@@ -1418,7 +1446,7 @@ void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
 
         iounmap(ring->virtual_start);
 
-        i915_gem_object_unpin(ring->obj);
+        i915_gem_object_ggtt_unpin(ring->obj);
         drm_gem_object_unreference(&ring->obj->base);
         ring->obj = NULL;
         ring->preallocated_lazy_request = NULL;
@@ -1430,28 +1458,16 @@ void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
         cleanup_status_page(ring);
 }
 
-static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
-{
-        int ret;
-
-        ret = i915_wait_seqno(ring, seqno);
-        if (!ret)
-                i915_gem_retire_requests_ring(ring);
-
-        return ret;
-}
-
 static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
 {
         struct drm_i915_gem_request *request;
-        u32 seqno = 0;
+        u32 seqno = 0, tail;
         int ret;
 
-        i915_gem_retire_requests_ring(ring);
-
         if (ring->last_retired_head != -1) {
                 ring->head = ring->last_retired_head;
                 ring->last_retired_head = -1;
+
                 ring->space = ring_space(ring);
                 if (ring->space >= n)
                         return 0;
@@ -1468,6 +1484,7 @@ static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
                         space += ring->size;
                 if (space >= n) {
                         seqno = request->seqno;
+                        tail = request->tail;
                         break;
                 }
 
@@ -1482,15 +1499,11 @@ static int intel_ring_wait_request(struct intel_ring_buffer *ring, int n)
         if (seqno == 0)
                 return -ENOSPC;
 
-        ret = intel_ring_wait_seqno(ring, seqno);
+        ret = i915_wait_seqno(ring, seqno);
         if (ret)
                 return ret;
 
-        if (WARN_ON(ring->last_retired_head == -1))
-                return -ENOSPC;
-
-        ring->head = ring->last_retired_head;
-        ring->last_retired_head = -1;
+        ring->head = tail;
         ring->space = ring_space(ring);
         if (WARN_ON(ring->space < n))
                 return -ENOSPC;
@@ -1528,7 +1541,8 @@ static int ring_wait_for_space(struct intel_ring_buffer *ring, int n)
                         return 0;
                 }
 
-                if (dev->primary->master) {
+                if (!drm_core_check_feature(dev, DRIVER_MODESET) &&
+                    dev->primary->master) {
                         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
                         if (master_priv->sarea_priv)
                                 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
@@ -1632,7 +1646,7 @@ static int __intel_ring_prepare(struct intel_ring_buffer *ring,
 int intel_ring_begin(struct intel_ring_buffer *ring,
                      int num_dwords)
 {
-        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
         int ret;
 
         ret = i915_gem_check_wedge(&dev_priv->gpu_error,
@@ -1694,7 +1708,7 @@ void intel_ring_init_seqno(struct intel_ring_buffer *ring, u32 seqno)
 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
                                      u32 value)
 {
-        drm_i915_private_t *dev_priv = ring->dev->dev_private;
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
 
        /* Every tail move must follow the sequence below */
 
@@ -1869,7 +1883,7 @@ static int gen6_ring_flush(struct intel_ring_buffer *ring,
 
 int intel_init_render_ring_buffer(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
 
         ring->name = "render ring";
@@ -1954,7 +1968,7 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
                         return -ENOMEM;
                 }
 
-                ret = i915_gem_obj_ggtt_pin(obj, 0, true, false);
+                ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
                 if (ret != 0) {
                         drm_gem_object_unreference(&obj->base);
                         DRM_ERROR("Failed to ping batch bo\n");
@@ -1970,7 +1984,7 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
 
 int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
         int ret;
 
@@ -2038,7 +2052,7 @@ int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
 
 int intel_init_bsd_ring_buffer(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
 
         ring->name = "bsd ring";
@@ -2101,7 +2115,7 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev)
 
 int intel_init_blt_ring_buffer(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
 
         ring->name = "blitter ring";
@@ -2141,7 +2155,7 @@ int intel_init_blt_ring_buffer(struct drm_device *dev)
 
 int intel_init_vebox_ring_buffer(struct drm_device *dev)
 {
-        drm_i915_private_t *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_ring_buffer *ring = &dev_priv->ring[VECS];
 
         ring->name = "video enhancement ring";
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 0b243ce33714..270a6a973438 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -33,6 +33,8 @@ struct  intel_hw_status_page {
 #define I915_READ_IMR(ring) I915_READ(RING_IMR((ring)->mmio_base))
 #define I915_WRITE_IMR(ring, val) I915_WRITE(RING_IMR((ring)->mmio_base), val)
 
+#define I915_READ_MODE(ring) I915_READ(RING_MI_MODE((ring)->mmio_base))
+
 enum intel_ring_hangcheck_action {
         HANGCHECK_IDLE = 0,
         HANGCHECK_WAIT,
@@ -41,12 +43,14 @@ enum intel_ring_hangcheck_action {
         HANGCHECK_HUNG,
 };
 
+#define HANGCHECK_SCORE_RING_HUNG 31
+
 struct intel_ring_hangcheck {
-        bool deadlock;
+        u64 acthd;
         u32 seqno;
-        u32 acthd;
         int score;
         enum intel_ring_hangcheck_action action;
+        bool deadlock;
 };
 
 struct  intel_ring_buffer {
@@ -162,6 +166,38 @@ struct  intel_ring_buffer {
                 u32 gtt_offset;
                 volatile u32 *cpu_page;
         } scratch;
+
+        /*
+         * Tables of commands the command parser needs to know about
+         * for this ring.
+         */
+        const struct drm_i915_cmd_table *cmd_tables;
+        int cmd_table_count;
+
+        /*
+         * Table of registers allowed in commands that read/write registers.
+         */
+        const u32 *reg_table;
+        int reg_count;
+
+        /*
+         * Table of registers allowed in commands that read/write registers, but
+         * only from the DRM master.
+         */
+        const u32 *master_reg_table;
+        int master_reg_count;
+
+        /*
+         * Returns the bitmask for the length field of the specified command.
+         * Return 0 for an unrecognized/invalid command.
+         *
+         * If the command parser finds an entry for a command in the ring's
+         * cmd_tables, it gets the command's length based on the table entry.
+         * If not, it calls this function to determine the per-ring length field
+         * encoding for the command (i.e. certain opcode ranges use certain bits
+         * to encode the command length in the header).
+         */
+        u32 (*get_cmd_length_mask)(u32 cmd_header);
 };
 
 static inline bool
@@ -256,7 +292,7 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev);
 int intel_init_blt_ring_buffer(struct drm_device *dev);
 int intel_init_vebox_ring_buffer(struct drm_device *dev);
 
-u32 intel_ring_get_active_head(struct intel_ring_buffer *ring);
+u64 intel_ring_get_active_head(struct intel_ring_buffer *ring);
 void intel_ring_setup_status_page(struct intel_ring_buffer *ring);
 
 static inline u32 intel_ring_get_tail(struct intel_ring_buffer *ring)
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index 95bdfb3c431c..d27155adf5db 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -1461,7 +1461,7 @@ static void intel_enable_sdvo(struct intel_encoder *encoder)
         u32 temp;
         bool input1, input2;
         int i;
-        u8 status;
+        bool success;
 
         temp = I915_READ(intel_sdvo->sdvo_reg);
         if ((temp & SDVO_ENABLE) == 0) {
@@ -1475,12 +1475,12 @@ static void intel_enable_sdvo(struct intel_encoder *encoder)
         for (i = 0; i < 2; i++)
                 intel_wait_for_vblank(dev, intel_crtc->pipe);
 
-        status = intel_sdvo_get_trained_inputs(intel_sdvo, &input1, &input2);
+        success = intel_sdvo_get_trained_inputs(intel_sdvo, &input1, &input2);
         /* Warn if the device reported failure to sync.
          * A lot of SDVO devices fail to notify of sync, but it's
          * a given it the status is a success, we succeeded.
          */
-        if (status == SDVO_CMD_STATUS_SUCCESS && !input1) {
+        if (success && !input1) {
                 DRM_DEBUG_KMS("First %s output reported failure to "
                                 "sync\n", SDVO_NAME(intel_sdvo));
         }
@@ -2382,24 +2382,62 @@ intel_sdvo_get_slave_addr(struct drm_device *dev, struct intel_sdvo *sdvo)
 }
 
 static void
+intel_sdvo_connector_unregister(struct intel_connector *intel_connector)
+{
+        struct drm_connector *drm_connector;
+        struct intel_sdvo *sdvo_encoder;
+
+        drm_connector = &intel_connector->base;
+        sdvo_encoder = intel_attached_sdvo(&intel_connector->base);
+
+        sysfs_remove_link(&drm_connector->kdev->kobj,
+                          sdvo_encoder->ddc.dev.kobj.name);
+        intel_connector_unregister(intel_connector);
+}
+
+static int
 intel_sdvo_connector_init(struct intel_sdvo_connector *connector,
                           struct intel_sdvo *encoder)
 {
-        drm_connector_init(encoder->base.base.dev,
-                           &connector->base.base,
+        struct drm_connector *drm_connector;
+        int ret;
+
+        drm_connector = &connector->base.base;
+        ret = drm_connector_init(encoder->base.base.dev,
+                           drm_connector,
                            &intel_sdvo_connector_funcs,
                            connector->base.base.connector_type);
+        if (ret < 0)
+                return ret;
 
-        drm_connector_helper_add(&connector->base.base,
+        drm_connector_helper_add(drm_connector,
                                  &intel_sdvo_connector_helper_funcs);
 
         connector->base.base.interlace_allowed = 1;
         connector->base.base.doublescan_allowed = 0;
         connector->base.base.display_info.subpixel_order = SubPixelHorizontalRGB;
         connector->base.get_hw_state = intel_sdvo_connector_get_hw_state;
+        connector->base.unregister = intel_sdvo_connector_unregister;
 
         intel_connector_attach_encoder(&connector->base, &encoder->base);
-        drm_sysfs_connector_add(&connector->base.base);
+        ret = drm_sysfs_connector_add(drm_connector);
+        if (ret < 0)
+                goto err1;
+
+        ret = sysfs_create_link(&encoder->ddc.dev.kobj,
+                                &drm_connector->kdev->kobj,
+                                encoder->ddc.dev.kobj.name);
+        if (ret < 0)
+                goto err2;
+
+        return 0;
+
+err2:
+        drm_sysfs_connector_remove(drm_connector);
+err1:
+        drm_connector_cleanup(drm_connector);
+
+        return ret;
 }
 
 static void
@@ -2459,7 +2497,11 @@ intel_sdvo_dvi_init(struct intel_sdvo *intel_sdvo, int device)
                 intel_sdvo->is_hdmi = true;
         }
 
-        intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo);
+        if (intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo) < 0) {
+                kfree(intel_sdvo_connector);
+                return false;
+        }
+
         if (intel_sdvo->is_hdmi)
                 intel_sdvo_add_hdmi_properties(intel_sdvo, intel_sdvo_connector);
 
@@ -2490,7 +2532,10 @@ intel_sdvo_tv_init(struct intel_sdvo *intel_sdvo, int type)
 
         intel_sdvo->is_tv = true;
 
-        intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo);
+        if (intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo) < 0) {
+                kfree(intel_sdvo_connector);
+                return false;
+        }
 
         if (!intel_sdvo_tv_create_property(intel_sdvo, intel_sdvo_connector, type))
                 goto err;
@@ -2534,8 +2579,11 @@ intel_sdvo_analog_init(struct intel_sdvo *intel_sdvo, int device)
                 intel_sdvo_connector->output_flag = SDVO_OUTPUT_RGB1;
         }
 
-        intel_sdvo_connector_init(intel_sdvo_connector,
-                                  intel_sdvo);
+        if (intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo) < 0) {
+                kfree(intel_sdvo_connector);
+                return false;
+        }
+
         return true;
 }
 
@@ -2566,7 +2614,11 @@ intel_sdvo_lvds_init(struct intel_sdvo *intel_sdvo, int device)
                 intel_sdvo_connector->output_flag = SDVO_OUTPUT_LVDS1;
         }
 
-        intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo);
+        if (intel_sdvo_connector_init(intel_sdvo_connector, intel_sdvo) < 0) {
+                kfree(intel_sdvo_connector);
+                return false;
+        }
+
         if (!intel_sdvo_create_enhance_property(intel_sdvo, intel_sdvo_connector))
                 goto err;
 
@@ -2980,7 +3032,7 @@ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob)
          * simplistic anyway to express such constraints, so just give up on
          * cloning for SDVO encoders.
          */
-        intel_sdvo->base.cloneable = false;
+        intel_sdvo->base.cloneable = 0;
 
         intel_sdvo_select_ddc_bus(dev_priv, intel_sdvo, sdvo_reg);
 
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 716a3c9c0751..336ae6c602f2 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -124,9 +124,6 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc,
         crtc_w--;
         crtc_h--;
 
-        I915_WRITE(SPSTRIDE(pipe, plane), fb->pitches[0]);
-        I915_WRITE(SPPOS(pipe, plane), (crtc_y << 16) | crtc_x);
-
         linear_offset = y * fb->pitches[0] + x * pixel_size;
         sprsurf_offset = intel_gen4_compute_page_offset(&x, &y,
                                                         obj->tiling_mode,
@@ -134,6 +131,9 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc,
                                                         fb->pitches[0]);
         linear_offset -= sprsurf_offset;
 
+        I915_WRITE(SPSTRIDE(pipe, plane), fb->pitches[0]);
+        I915_WRITE(SPPOS(pipe, plane), (crtc_y << 16) | crtc_x);
+
         if (obj->tiling_mode != I915_TILING_NONE)
                 I915_WRITE(SPTILEOFF(pipe, plane), (y << 16) | x);
         else
@@ -293,15 +293,15 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
         if (crtc_w != src_w || crtc_h != src_h)
                 sprscale = SPRITE_SCALE_ENABLE | (src_w << 16) | src_h;
 
-        I915_WRITE(SPRSTRIDE(pipe), fb->pitches[0]);
-        I915_WRITE(SPRPOS(pipe), (crtc_y << 16) | crtc_x);
-
         linear_offset = y * fb->pitches[0] + x * pixel_size;
         sprsurf_offset =
                 intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode,
                                                pixel_size, fb->pitches[0]);
         linear_offset -= sprsurf_offset;
 
+        I915_WRITE(SPRSTRIDE(pipe), fb->pitches[0]);
+        I915_WRITE(SPRPOS(pipe), (crtc_y << 16) | crtc_x);
+
         /* HSW consolidates SPRTILEOFF and SPRLINOFF into a single SPROFFSET
          * register */
         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
@@ -472,15 +472,15 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
         if (crtc_w != src_w || crtc_h != src_h)
                 dvsscale = DVS_SCALE_ENABLE | (src_w << 16) | src_h;
 
-        I915_WRITE(DVSSTRIDE(pipe), fb->pitches[0]);
-        I915_WRITE(DVSPOS(pipe), (crtc_y << 16) | crtc_x);
-
         linear_offset = y * fb->pitches[0] + x * pixel_size;
         dvssurf_offset =
                 intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode,
                                                pixel_size, fb->pitches[0]);
         linear_offset -= dvssurf_offset;
 
+        I915_WRITE(DVSSTRIDE(pipe), fb->pitches[0]);
+        I915_WRITE(DVSPOS(pipe), (crtc_y << 16) | crtc_x);
+
         if (obj->tiling_mode != I915_TILING_NONE)
                 I915_WRITE(DVSTILEOFF(pipe), (y << 16) | x);
         else
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index 22cf0f4ba248..bafe92e317d5 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -1189,8 +1189,8 @@ intel_tv_detect_type(struct intel_tv *intel_tv,
         if (connector->polled & DRM_CONNECTOR_POLL_HPD) {
                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
                 i915_disable_pipestat(dev_priv, 0,
-                                      PIPE_HOTPLUG_INTERRUPT_ENABLE |
-                                      PIPE_HOTPLUG_TV_INTERRUPT_ENABLE);
+                                      PIPE_HOTPLUG_INTERRUPT_STATUS |
+                                      PIPE_HOTPLUG_TV_INTERRUPT_STATUS);
                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
         }
 
@@ -1266,8 +1266,8 @@ intel_tv_detect_type(struct intel_tv *intel_tv,
         if (connector->polled & DRM_CONNECTOR_POLL_HPD) {
                 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
                 i915_enable_pipestat(dev_priv, 0,
-                                     PIPE_HOTPLUG_INTERRUPT_ENABLE |
-                                     PIPE_HOTPLUG_TV_INTERRUPT_ENABLE);
+                                     PIPE_HOTPLUG_INTERRUPT_STATUS |
+                                     PIPE_HOTPLUG_TV_INTERRUPT_STATUS);
                 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
         }
 
@@ -1536,9 +1536,14 @@ static int tv_is_present_in_vbt(struct drm_device *dev)
                 /*
                  * If the device type is not TV, continue.
                  */
-                if (p_child->old.device_type != DEVICE_TYPE_INT_TV &&
-                        p_child->old.device_type != DEVICE_TYPE_TV)
+                switch (p_child->old.device_type) {
+                case DEVICE_TYPE_INT_TV:
+                case DEVICE_TYPE_TV:
+                case DEVICE_TYPE_TV_SVIDEO_COMPOSITE:
+                        break;
+                default:
                         continue;
+                }
                 /* Only when the addin_offset is non-zero, it is regarded
                  * as present.
                  */
@@ -1634,13 +1639,13 @@ intel_tv_init(struct drm_device *dev)
         intel_encoder->disable = intel_disable_tv;
         intel_encoder->get_hw_state = intel_tv_get_hw_state;
         intel_connector->get_hw_state = intel_connector_get_hw_state;
+        intel_connector->unregister = intel_connector_unregister;
 
         intel_connector_attach_encoder(intel_connector, intel_encoder);
         intel_encoder->type = INTEL_OUTPUT_TVOUT;
         intel_encoder->crtc_mask = (1 << 0) | (1 << 1);
-        intel_encoder->cloneable = false;
+        intel_encoder->cloneable = 0;
         intel_encoder->base.possible_crtcs = ((1 << 0) | (1 << 1));
-        intel_encoder->base.possible_clones = (1 << INTEL_OUTPUT_TVOUT);
         intel_tv->type = DRM_MODE_CONNECTOR_Unknown;
 
         /* BIOS margin values */
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 87df68f5f504..f729dc71d5be 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -40,6 +40,12 @@
 
 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32(dev_priv__, reg__)
 
+static void
+assert_device_not_suspended(struct drm_i915_private *dev_priv)
+{
+        WARN(HAS_RUNTIME_PM(dev_priv->dev) && dev_priv->pm.suspended,
+             "Device suspended\n");
+}
 
 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
 {
@@ -83,14 +89,14 @@ static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv,
         __gen6_gt_wait_for_thread_c0(dev_priv);
 }
 
-static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
+static void __gen7_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv)
 {
         __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff));
         /* something from same cacheline, but !FORCEWAKE_MT */
         __raw_posting_read(dev_priv, ECOBUS);
 }
 
-static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv,
+static void __gen7_gt_force_wake_mt_get(struct drm_i915_private *dev_priv,
                                                         int fw_engine)
 {
         u32 forcewake_ack;
@@ -136,14 +142,16 @@ static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv,
         gen6_gt_check_fifodbg(dev_priv);
 }
 
-static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv,
+static void __gen7_gt_force_wake_mt_put(struct drm_i915_private *dev_priv,
                                                         int fw_engine)
 {
         __raw_i915_write32(dev_priv, FORCEWAKE_MT,
                            _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL));
         /* something from same cacheline, but !FORCEWAKE_MT */
         __raw_posting_read(dev_priv, ECOBUS);
-        gen6_gt_check_fifodbg(dev_priv);
+
+        if (IS_GEN7(dev_priv->dev))
+                gen6_gt_check_fifodbg(dev_priv);
 }
 
 static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
@@ -251,16 +259,16 @@ void vlv_force_wake_get(struct drm_i915_private *dev_priv,
         unsigned long irqflags;
 
         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
-        if (FORCEWAKE_RENDER & fw_engine) {
-                if (dev_priv->uncore.fw_rendercount++ == 0)
-                        dev_priv->uncore.funcs.force_wake_get(dev_priv,
-                                                        FORCEWAKE_RENDER);
-        }
-        if (FORCEWAKE_MEDIA & fw_engine) {
-                if (dev_priv->uncore.fw_mediacount++ == 0)
-                        dev_priv->uncore.funcs.force_wake_get(dev_priv,
-                                                        FORCEWAKE_MEDIA);
-        }
+
+        if (fw_engine & FORCEWAKE_RENDER &&
+            dev_priv->uncore.fw_rendercount++ != 0)
+                fw_engine &= ~FORCEWAKE_RENDER;
+        if (fw_engine & FORCEWAKE_MEDIA &&
+            dev_priv->uncore.fw_mediacount++ != 0)
+                fw_engine &= ~FORCEWAKE_MEDIA;
+
+        if (fw_engine)
+                dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_engine);
 
         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
@@ -272,46 +280,89 @@ void vlv_force_wake_put(struct drm_i915_private *dev_priv,
 
         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
-        if (FORCEWAKE_RENDER & fw_engine) {
-                WARN_ON(dev_priv->uncore.fw_rendercount == 0);
-                if (--dev_priv->uncore.fw_rendercount == 0)
-                        dev_priv->uncore.funcs.force_wake_put(dev_priv,
-                                                        FORCEWAKE_RENDER);
+        if (fw_engine & FORCEWAKE_RENDER) {
+                WARN_ON(!dev_priv->uncore.fw_rendercount);
+                if (--dev_priv->uncore.fw_rendercount != 0)
+                        fw_engine &= ~FORCEWAKE_RENDER;
         }
 
-        if (FORCEWAKE_MEDIA & fw_engine) {
-                WARN_ON(dev_priv->uncore.fw_mediacount == 0);
-                if (--dev_priv->uncore.fw_mediacount == 0)
-                        dev_priv->uncore.funcs.force_wake_put(dev_priv,
-                                                        FORCEWAKE_MEDIA);
+        if (fw_engine & FORCEWAKE_MEDIA) {
+                WARN_ON(!dev_priv->uncore.fw_mediacount);
+                if (--dev_priv->uncore.fw_mediacount != 0)
+                        fw_engine &= ~FORCEWAKE_MEDIA;
         }
 
+        if (fw_engine)
+                dev_priv->uncore.funcs.force_wake_put(dev_priv, fw_engine);
+
         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
 
-static void gen6_force_wake_work(struct work_struct *work)
+static void gen6_force_wake_timer(unsigned long arg)
 {
-        struct drm_i915_private *dev_priv =
-                container_of(work, typeof(*dev_priv), uncore.force_wake_work.work);
+        struct drm_i915_private *dev_priv = (void *)arg;
         unsigned long irqflags;
 
+        assert_device_not_suspended(dev_priv);
+
         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
+        WARN_ON(!dev_priv->uncore.forcewake_count);
+
         if (--dev_priv->uncore.forcewake_count == 0)
                 dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
+
+        intel_runtime_pm_put(dev_priv);
 }
 
-static void intel_uncore_forcewake_reset(struct drm_device *dev)
+static void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long irqflags;
 
-        if (IS_VALLEYVIEW(dev)) {
+        del_timer_sync(&dev_priv->uncore.force_wake_timer);
+
+        /* Hold uncore.lock across reset to prevent any register access
+         * with forcewake not set correctly
+         */
+        spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
+
+        if (IS_VALLEYVIEW(dev))
                 vlv_force_wake_reset(dev_priv);
-        } else if (INTEL_INFO(dev)->gen >= 6) {
+        else if (IS_GEN6(dev) || IS_GEN7(dev))
                 __gen6_gt_force_wake_reset(dev_priv);
-                if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
-                        __gen6_gt_force_wake_mt_reset(dev_priv);
+
+        if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_GEN8(dev))
+                __gen7_gt_force_wake_mt_reset(dev_priv);
+
+        if (restore) { /* If reset with a user forcewake, try to restore */
+                unsigned fw = 0;
+
+                if (IS_VALLEYVIEW(dev)) {
+                        if (dev_priv->uncore.fw_rendercount)
+                                fw |= FORCEWAKE_RENDER;
+
+                        if (dev_priv->uncore.fw_mediacount)
+                                fw |= FORCEWAKE_MEDIA;
+                } else {
+                        if (dev_priv->uncore.forcewake_count)
+                                fw = FORCEWAKE_ALL;
+                }
+
+                if (fw)
+                        dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);
+
+                if (IS_GEN6(dev) || IS_GEN7(dev))
+                        dev_priv->uncore.fifo_count =
+                                __raw_i915_read32(dev_priv, GTFIFOCTL) &
+                                GT_FIFO_FREE_ENTRIES_MASK;
+        } else {
+                dev_priv->uncore.forcewake_count = 0;
+                dev_priv->uncore.fw_rendercount = 0;
+                dev_priv->uncore.fw_mediacount = 0;
         }
+
+        spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 }
 
 void intel_uncore_early_sanitize(struct drm_device *dev)
@@ -337,7 +388,7 @@ void intel_uncore_early_sanitize(struct drm_device *dev)
                 __raw_i915_write32(dev_priv, GTFIFODBG,
                                    __raw_i915_read32(dev_priv, GTFIFODBG));
 
-        intel_uncore_forcewake_reset(dev);
+        intel_uncore_forcewake_reset(dev, false);
 }
 
 void intel_uncore_sanitize(struct drm_device *dev)
@@ -354,7 +405,9 @@ void intel_uncore_sanitize(struct drm_device *dev)
                 mutex_lock(&dev_priv->rps.hw_lock);
                 reg_val = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS);
 
-                if (reg_val & (RENDER_PWRGT | MEDIA_PWRGT | DISP2D_PWRGT))
+                if (reg_val & (PUNIT_PWRGT_PWR_GATE(PUNIT_POWER_WELL_RENDER) |
+                               PUNIT_PWRGT_PWR_GATE(PUNIT_POWER_WELL_MEDIA) |
+                               PUNIT_PWRGT_PWR_GATE(PUNIT_POWER_WELL_DISP2D)))
                         vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, 0x0);
 
                 mutex_unlock(&dev_priv->rps.hw_lock);
@@ -393,25 +446,40 @@ void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine)
 void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine)
 {
         unsigned long irqflags;
+        bool delayed = false;
 
         if (!dev_priv->uncore.funcs.force_wake_put)
                 return;
 
         /* Redirect to VLV specific routine */
-        if (IS_VALLEYVIEW(dev_priv->dev))
-                return vlv_force_wake_put(dev_priv, fw_engine);
+        if (IS_VALLEYVIEW(dev_priv->dev)) {
+                vlv_force_wake_put(dev_priv, fw_engine);
+                goto out;
+        }
 
 
         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
+        WARN_ON(!dev_priv->uncore.forcewake_count);
+
         if (--dev_priv->uncore.forcewake_count == 0) {
                 dev_priv->uncore.forcewake_count++;
-                mod_delayed_work(dev_priv->wq,
-                                 &dev_priv->uncore.force_wake_work,
-                                 1);
+                delayed = true;
+                mod_timer_pinned(&dev_priv->uncore.force_wake_timer,
+                                 jiffies + 1);
         }
         spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
 
-        intel_runtime_pm_put(dev_priv);
+out:
+        if (!delayed)
+                intel_runtime_pm_put(dev_priv);
+}
+
+void assert_force_wake_inactive(struct drm_i915_private *dev_priv)
+{
+        if (!dev_priv->uncore.funcs.force_wake_get)
+                return;
+
+        WARN_ON(dev_priv->uncore.forcewake_count > 0);
 }
 
 /* We give fast paths for the really cool registers */
@@ -446,16 +514,10 @@ hsw_unclaimed_reg_check(struct drm_i915_private *dev_priv, u32 reg)
         }
 }
 
-static void
-assert_device_not_suspended(struct drm_i915_private *dev_priv)
-{
-        WARN(HAS_RUNTIME_PM(dev_priv->dev) && dev_priv->pm.suspended,
-             "Device suspended\n");
-}
-
 #define REG_READ_HEADER(x) \
         unsigned long irqflags; \
         u##x val = 0; \
+        assert_device_not_suspended(dev_priv); \
         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 
 #define REG_READ_FOOTER \
@@ -484,14 +546,13 @@ gen5_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 static u##x \
 gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
         REG_READ_HEADER(x); \
-        if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
-                if (dev_priv->uncore.forcewake_count == 0) \
-                        dev_priv->uncore.funcs.force_wake_get(dev_priv, \
-                                                        FORCEWAKE_ALL); \
+        if (dev_priv->uncore.forcewake_count == 0 && \
+            NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
+                dev_priv->uncore.funcs.force_wake_get(dev_priv, \
+                                                      FORCEWAKE_ALL); \
                 val = __raw_i915_read##x(dev_priv, reg); \
-                if (dev_priv->uncore.forcewake_count == 0) \
-                        dev_priv->uncore.funcs.force_wake_put(dev_priv, \
-                                                        FORCEWAKE_ALL); \
+                dev_priv->uncore.funcs.force_wake_put(dev_priv, \
+                                                      FORCEWAKE_ALL); \
         } else { \
                 val = __raw_i915_read##x(dev_priv, reg); \
         } \
@@ -502,27 +563,19 @@ gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
 static u##x \
 vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \
         unsigned fwengine = 0; \
-        unsigned *fwcount; \
         REG_READ_HEADER(x); \
-        if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) {   \
-                fwengine = FORCEWAKE_RENDER;            \
-                fwcount = &dev_priv->uncore.fw_rendercount;    \
-        }                                               \
-        else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) {       \
-                fwengine = FORCEWAKE_MEDIA;             \
-                fwcount = &dev_priv->uncore.fw_mediacount;     \
+        if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) { \
+                if (dev_priv->uncore.fw_rendercount == 0) \
+                        fwengine = FORCEWAKE_RENDER; \
+        } else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) { \
+                if (dev_priv->uncore.fw_mediacount == 0) \
+                        fwengine = FORCEWAKE_MEDIA; \
         }  \
-        if (fwengine != 0) {            \
-                if ((*fwcount)++ == 0) \
-                        (dev_priv)->uncore.funcs.force_wake_get(dev_priv, \
-                                                                fwengine); \
-                val = __raw_i915_read##x(dev_priv, reg); \
-                if (--(*fwcount) == 0) \
-                        (dev_priv)->uncore.funcs.force_wake_put(dev_priv, \
-                                                        fwengine); \
-        } else { \
-                val = __raw_i915_read##x(dev_priv, reg); \
-        } \
+        if (fwengine) \
+                dev_priv->uncore.funcs.force_wake_get(dev_priv, fwengine); \
+        val = __raw_i915_read##x(dev_priv, reg); \
+        if (fwengine) \
+                dev_priv->uncore.funcs.force_wake_put(dev_priv, fwengine); \
         REG_READ_FOOTER; \
 }
 
@@ -554,6 +607,7 @@ __gen4_read(64)
 #define REG_WRITE_HEADER \
         unsigned long irqflags; \
         trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
+        assert_device_not_suspended(dev_priv); \
         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags)
 
 #define REG_WRITE_FOOTER \
@@ -584,7 +638,6 @@ gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace
         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
         } \
-        assert_device_not_suspended(dev_priv); \
         __raw_i915_write##x(dev_priv, reg, val); \
         if (unlikely(__fifo_ret)) { \
                 gen6_gt_check_fifodbg(dev_priv); \
@@ -600,7 +653,6 @@ hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace)
         if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \
                 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \
         } \
-        assert_device_not_suspended(dev_priv); \
         hsw_unclaimed_reg_clear(dev_priv, reg); \
         __raw_i915_write##x(dev_priv, reg, val); \
         if (unlikely(__fifo_ret)) { \
@@ -634,16 +686,17 @@ static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg)
 #define __gen8_write(x) \
 static void \
 gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \
-        bool __needs_put = reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg); \
         REG_WRITE_HEADER; \
-        if (__needs_put) { \
-                dev_priv->uncore.funcs.force_wake_get(dev_priv, \
-                                                        FORCEWAKE_ALL); \
-        } \
-        __raw_i915_write##x(dev_priv, reg, val); \
-        if (__needs_put) { \
-                dev_priv->uncore.funcs.force_wake_put(dev_priv, \
-                                                        FORCEWAKE_ALL); \
+        if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) { \
+                if (dev_priv->uncore.forcewake_count == 0) \
+                        dev_priv->uncore.funcs.force_wake_get(dev_priv, \
+                                                              FORCEWAKE_ALL); \
+                __raw_i915_write##x(dev_priv, reg, val); \
+                if (dev_priv->uncore.forcewake_count == 0) \
+                        dev_priv->uncore.funcs.force_wake_put(dev_priv, \
+                                                              FORCEWAKE_ALL); \
+        } else { \
+                __raw_i915_write##x(dev_priv, reg, val); \
         } \
         REG_WRITE_FOOTER; \
 }
@@ -681,15 +734,17 @@ void intel_uncore_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        INIT_DELAYED_WORK(&dev_priv->uncore.force_wake_work,
-                          gen6_force_wake_work);
+        setup_timer(&dev_priv->uncore.force_wake_timer,
+                    gen6_force_wake_timer, (unsigned long)dev_priv);
+
+        intel_uncore_early_sanitize(dev);
 
         if (IS_VALLEYVIEW(dev)) {
                 dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get;
                 dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put;
         } else if (IS_HASWELL(dev) || IS_GEN8(dev)) {
-                dev_priv->uncore.funcs.force_wake_get = __gen6_gt_force_wake_mt_get;
-                dev_priv->uncore.funcs.force_wake_put = __gen6_gt_force_wake_mt_put;
+                dev_priv->uncore.funcs.force_wake_get = __gen7_gt_force_wake_mt_get;
+                dev_priv->uncore.funcs.force_wake_put = __gen7_gt_force_wake_mt_put;
         } else if (IS_IVYBRIDGE(dev)) {
                 u32 ecobus;
 
@@ -703,16 +758,16 @@ void intel_uncore_init(struct drm_device *dev)
                  * forcewake being disabled.
                  */
                 mutex_lock(&dev->struct_mutex);
-                __gen6_gt_force_wake_mt_get(dev_priv, FORCEWAKE_ALL);
+                __gen7_gt_force_wake_mt_get(dev_priv, FORCEWAKE_ALL);
                 ecobus = __raw_i915_read32(dev_priv, ECOBUS);
-                __gen6_gt_force_wake_mt_put(dev_priv, FORCEWAKE_ALL);
+                __gen7_gt_force_wake_mt_put(dev_priv, FORCEWAKE_ALL);
                 mutex_unlock(&dev->struct_mutex);
 
                 if (ecobus & FORCEWAKE_MT_ENABLE) {
                         dev_priv->uncore.funcs.force_wake_get =
-                                __gen6_gt_force_wake_mt_get;
+                                __gen7_gt_force_wake_mt_get;
                         dev_priv->uncore.funcs.force_wake_put =
-                                __gen6_gt_force_wake_mt_put;
+                                __gen7_gt_force_wake_mt_put;
                 } else {
                         DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
                         DRM_INFO("when using vblank-synced partial screen updates.\n");
@@ -792,12 +847,9 @@ void intel_uncore_init(struct drm_device *dev)
 
 void intel_uncore_fini(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        flush_delayed_work(&dev_priv->uncore.force_wake_work);
-
         /* Paranoia: make sure we have disabled everything before we exit. */
         intel_uncore_sanitize(dev);
+        intel_uncore_forcewake_reset(dev, false);
 }
 
 static const struct register_whitelist {
@@ -814,7 +866,7 @@ int i915_reg_read_ioctl(struct drm_device *dev,
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_reg_read *reg = data;
         struct register_whitelist const *entry = whitelist;
-        int i;
+        int i, ret = 0;
 
         for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
                 if (entry->offset == reg->offset &&
@@ -825,6 +877,8 @@ int i915_reg_read_ioctl(struct drm_device *dev,
         if (i == ARRAY_SIZE(whitelist))
                 return -EINVAL;
 
+        intel_runtime_pm_get(dev_priv);
+
         switch (entry->size) {
         case 8:
                 reg->val = I915_READ64(reg->offset);
@@ -840,10 +894,13 @@ int i915_reg_read_ioctl(struct drm_device *dev,
                 break;
         default:
                 WARN_ON(1);
-                return -EINVAL;
+                ret = -EINVAL;
+                goto out;
         }
 
-        return 0;
+out:
+        intel_runtime_pm_put(dev_priv);
+        return ret;
 }
 
 int i915_get_reset_stats_ioctl(struct drm_device *dev,
@@ -852,6 +909,7 @@ int i915_get_reset_stats_ioctl(struct drm_device *dev,
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_reset_stats *args = data;
         struct i915_ctx_hang_stats *hs;
+        struct i915_hw_context *ctx;
         int ret;
 
         if (args->flags || args->pad)
@@ -864,11 +922,12 @@ int i915_get_reset_stats_ioctl(struct drm_device *dev,
         if (ret)
                 return ret;
 
-        hs = i915_gem_context_get_hang_stats(dev, file, args->ctx_id);
-        if (IS_ERR(hs)) {
+        ctx = i915_gem_context_get(file->driver_priv, args->ctx_id);
+        if (IS_ERR(ctx)) {
                 mutex_unlock(&dev->struct_mutex);
-                return PTR_ERR(hs);
+                return PTR_ERR(ctx);
         }
+        hs = &ctx->hang_stats;
 
         if (capable(CAP_SYS_ADMIN))
                 args->reset_count = i915_reset_count(&dev_priv->gpu_error);
@@ -944,12 +1003,6 @@ static int gen6_do_reset(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         int     ret;
-        unsigned long irqflags;
-
-        /* Hold uncore.lock across reset to prevent any register access
-         * with forcewake not set correctly
-         */
-        spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
         /* Reset the chip */
 
@@ -962,18 +1015,8 @@ static int gen6_do_reset(struct drm_device *dev)
         /* Spin waiting for the device to ack the reset request */
         ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500);
 
-        intel_uncore_forcewake_reset(dev);
+        intel_uncore_forcewake_reset(dev, true);
 
-        /* If reset with a user forcewake, try to restore, otherwise turn it off */
-        if (dev_priv->uncore.forcewake_count)
-                dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL);
-        else
-                dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL);
-
-        /* Restore fifo count */
-        dev_priv->uncore.fifo_count = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK;
-
-        spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
         return ret;
 }
 
diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
index 968374776db9..a034ed408252 100644
--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
+++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
@@ -29,7 +29,7 @@ static void mga_crtc_load_lut(struct drm_crtc *crtc)
         struct mga_crtc *mga_crtc = to_mga_crtc(crtc);
         struct drm_device *dev = crtc->dev;
         struct mga_device *mdev = dev->dev_private;
-        struct drm_framebuffer *fb = crtc->fb;
+        struct drm_framebuffer *fb = crtc->primary->fb;
         int i;
 
         if (!crtc->enabled)
@@ -742,7 +742,7 @@ static int mga_crtc_do_set_base(struct drm_crtc *crtc,
                 mgag200_bo_unreserve(bo);
         }
 
-        mga_fb = to_mga_framebuffer(crtc->fb);
+        mga_fb = to_mga_framebuffer(crtc->primary->fb);
         obj = mga_fb->obj;
         bo = gem_to_mga_bo(obj);
 
@@ -805,7 +805,7 @@ static int mga_crtc_mode_set(struct drm_crtc *crtc,
                 /* 0x48: */        0,    0,    0,    0,    0,    0,    0,    0
         };
 
-        bppshift = mdev->bpp_shifts[(crtc->fb->bits_per_pixel >> 3) - 1];
+        bppshift = mdev->bpp_shifts[(crtc->primary->fb->bits_per_pixel >> 3) - 1];
 
         switch (mdev->type) {
         case G200_SE_A:
@@ -843,12 +843,12 @@ static int mga_crtc_mode_set(struct drm_crtc *crtc,
                 break;
         }
 
-        switch (crtc->fb->bits_per_pixel) {
+        switch (crtc->primary->fb->bits_per_pixel) {
         case 8:
                 dacvalue[MGA1064_MUL_CTL] = MGA1064_MUL_CTL_8bits;
                 break;
         case 16:
-                if (crtc->fb->depth == 15)
+                if (crtc->primary->fb->depth == 15)
                         dacvalue[MGA1064_MUL_CTL] = MGA1064_MUL_CTL_15bits;
                 else
                         dacvalue[MGA1064_MUL_CTL] = MGA1064_MUL_CTL_16bits;
@@ -896,8 +896,8 @@ static int mga_crtc_mode_set(struct drm_crtc *crtc,
         WREG_SEQ(3, 0);
         WREG_SEQ(4, 0xe);
 
-        pitch = crtc->fb->pitches[0] / (crtc->fb->bits_per_pixel / 8);
-        if (crtc->fb->bits_per_pixel == 24)
+        pitch = crtc->primary->fb->pitches[0] / (crtc->primary->fb->bits_per_pixel / 8);
+        if (crtc->primary->fb->bits_per_pixel == 24)
                 pitch = (pitch * 3) >> (4 - bppshift);
         else
                 pitch = pitch >> (4 - bppshift);
@@ -974,7 +974,7 @@ static int mga_crtc_mode_set(struct drm_crtc *crtc,
                 ((vdisplay & 0xc00) >> 7) |
                 ((vsyncstart & 0xc00) >> 5) |
                 ((vdisplay & 0x400) >> 3);
-        if (crtc->fb->bits_per_pixel == 24)
+        if (crtc->primary->fb->bits_per_pixel == 24)
                 ext_vga[3] = (((1 << bppshift) * 3) - 1) | 0x80;
         else
                 ext_vga[3] = ((1 << bppshift) - 1) | 0x80;
@@ -1034,9 +1034,9 @@ static int mga_crtc_mode_set(struct drm_crtc *crtc,
                         u32 bpp;
                         u32 mb;
 
-                        if (crtc->fb->bits_per_pixel > 16)
+                        if (crtc->primary->fb->bits_per_pixel > 16)
                                 bpp = 32;
-                        else if (crtc->fb->bits_per_pixel > 8)
+                        else if (crtc->primary->fb->bits_per_pixel > 8)
                                 bpp = 16;
                         else
                                 bpp = 8;
@@ -1277,8 +1277,8 @@ static void mga_crtc_disable(struct drm_crtc *crtc)
         int ret;
         DRM_DEBUG_KMS("\n");
         mga_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
-        if (crtc->fb) {
-                struct mga_framebuffer *mga_fb = to_mga_framebuffer(crtc->fb);
+        if (crtc->primary->fb) {
+                struct mga_framebuffer *mga_fb = to_mga_framebuffer(crtc->primary->fb);
                 struct drm_gem_object *obj = mga_fb->obj;
                 struct mgag200_bo *bo = gem_to_mga_bo(obj);
                 ret = mgag200_bo_reserve(bo, false);
@@ -1287,7 +1287,7 @@ static void mga_crtc_disable(struct drm_crtc *crtc)
                 mgag200_bo_push_sysram(bo);
                 mgag200_bo_unreserve(bo);
         }
-        crtc->fb = NULL;
+        crtc->primary->fb = NULL;
 }
 
 /* These provide the minimum set of functions required to handle a CRTC */
diff --git a/drivers/gpu/drm/mgag200/mgag200_ttm.c b/drivers/gpu/drm/mgag200/mgag200_ttm.c
index adb5166a5dfd..5a00e90696de 100644
--- a/drivers/gpu/drm/mgag200/mgag200_ttm.c
+++ b/drivers/gpu/drm/mgag200/mgag200_ttm.c
@@ -259,7 +259,9 @@ int mgag200_mm_init(struct mga_device *mdev)
 
         ret = ttm_bo_device_init(&mdev->ttm.bdev,
                                  mdev->ttm.bo_global_ref.ref.object,
-                                 &mgag200_bo_driver, DRM_FILE_PAGE_OFFSET,
+                                 &mgag200_bo_driver,
+                                 dev->anon_inode->i_mapping,
+                                 DRM_FILE_PAGE_OFFSET,
                                  true);
         if (ret) {
                 DRM_ERROR("Error initialising bo driver; %d\n", ret);
@@ -324,7 +326,6 @@ int mgag200_bo_create(struct drm_device *dev, int size, int align,
         }
 
         mgabo->bo.bdev = &mdev->ttm.bdev;
-        mgabo->bo.bdev->dev_mapping = dev->dev_mapping;
 
         mgag200_ttm_placement(mgabo, TTM_PL_FLAG_VRAM | TTM_PL_FLAG_SYSTEM);
 
diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
index 4f977a593bea..5e1e6b0cd8ac 100644
--- a/drivers/gpu/drm/msm/Makefile
+++ b/drivers/gpu/drm/msm/Makefile
@@ -7,6 +7,7 @@ msm-y := \
         adreno/adreno_gpu.o \
         adreno/a3xx_gpu.o \
         hdmi/hdmi.o \
+        hdmi/hdmi_audio.o \
         hdmi/hdmi_bridge.o \
         hdmi/hdmi_connector.o \
         hdmi/hdmi_i2c.o \
diff --git a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
index 461df93e825e..f20fbde5dc49 100644
--- a/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a3xx_gpu.c
@@ -35,7 +35,11 @@
          A3XX_INT0_CP_AHB_ERROR_HALT |     \
          A3XX_INT0_UCHE_OOB_ACCESS)
 
-static struct platform_device *a3xx_pdev;
+
+static bool hang_debug = false;
+MODULE_PARM_DESC(hang_debug, "Dump registers when hang is detected (can be slow!)");
+module_param_named(hang_debug, hang_debug, bool, 0600);
+static void a3xx_dump(struct msm_gpu *gpu);
 
 static void a3xx_me_init(struct msm_gpu *gpu)
 {
@@ -291,6 +295,9 @@ static int a3xx_hw_init(struct msm_gpu *gpu)
 
 static void a3xx_recover(struct msm_gpu *gpu)
 {
+        /* dump registers before resetting gpu, if enabled: */
+        if (hang_debug)
+                a3xx_dump(gpu);
         gpu_write(gpu, REG_A3XX_RBBM_SW_RESET_CMD, 1);
         gpu_read(gpu, REG_A3XX_RBBM_SW_RESET_CMD);
         gpu_write(gpu, REG_A3XX_RBBM_SW_RESET_CMD, 0);
@@ -311,27 +318,18 @@ static void a3xx_destroy(struct msm_gpu *gpu)
                 ocmem_free(OCMEM_GRAPHICS, a3xx_gpu->ocmem_hdl);
 #endif
 
-        put_device(&a3xx_gpu->pdev->dev);
         kfree(a3xx_gpu);
 }
 
 static void a3xx_idle(struct msm_gpu *gpu)
 {
-        unsigned long t;
-
         /* wait for ringbuffer to drain: */
         adreno_idle(gpu);
 
-        t = jiffies + ADRENO_IDLE_TIMEOUT;
-
         /* then wait for GPU to finish: */
-        do {
-                uint32_t rbbm_status = gpu_read(gpu, REG_A3XX_RBBM_STATUS);
-                if (!(rbbm_status & A3XX_RBBM_STATUS_GPU_BUSY))
-                        return;
-        } while(time_before(jiffies, t));
-
-        DRM_ERROR("timeout waiting for %s to idle!\n", gpu->name);
+        if (spin_until(!(gpu_read(gpu, REG_A3XX_RBBM_STATUS) &
+                        A3XX_RBBM_STATUS_GPU_BUSY)))
+                DRM_ERROR("%s: timeout waiting for GPU to idle!\n", gpu->name);
 
         /* TODO maybe we need to reset GPU here to recover from hang? */
 }
@@ -352,7 +350,6 @@ static irqreturn_t a3xx_irq(struct msm_gpu *gpu)
         return IRQ_HANDLED;
 }
 
-#ifdef CONFIG_DEBUG_FS
 static const unsigned int a3xx_registers[] = {
         0x0000, 0x0002, 0x0010, 0x0012, 0x0018, 0x0018, 0x0020, 0x0027,
         0x0029, 0x002b, 0x002e, 0x0033, 0x0040, 0x0042, 0x0050, 0x005c,
@@ -392,11 +389,18 @@ static const unsigned int a3xx_registers[] = {
         0x303c, 0x303c, 0x305e, 0x305f,
 };
 
+#ifdef CONFIG_DEBUG_FS
 static void a3xx_show(struct msm_gpu *gpu, struct seq_file *m)
 {
+        struct drm_device *dev = gpu->dev;
         int i;
 
         adreno_show(gpu, m);
+
+        mutex_lock(&dev->struct_mutex);
+
+        gpu->funcs->pm_resume(gpu);
+
         seq_printf(m, "status:   %08x\n",
                         gpu_read(gpu, REG_A3XX_RBBM_STATUS));
 
@@ -412,9 +416,36 @@ static void a3xx_show(struct msm_gpu *gpu, struct seq_file *m)
                         seq_printf(m, "IO:R %08x %08x\n", addr<<2, val);
                 }
         }
+
+        gpu->funcs->pm_suspend(gpu);
+
+        mutex_unlock(&dev->struct_mutex);
 }
 #endif
 
+/* would be nice to not have to duplicate the _show() stuff with printk(): */
+static void a3xx_dump(struct msm_gpu *gpu)
+{
+        int i;
+
+        adreno_dump(gpu);
+        printk("status:   %08x\n",
+                        gpu_read(gpu, REG_A3XX_RBBM_STATUS));
+
+        /* dump these out in a form that can be parsed by demsm: */
+        printk("IO:region %s 00000000 00020000\n", gpu->name);
+        for (i = 0; i < ARRAY_SIZE(a3xx_registers); i += 2) {
+                uint32_t start = a3xx_registers[i];
+                uint32_t end   = a3xx_registers[i+1];
+                uint32_t addr;
+
+                for (addr = start; addr <= end; addr++) {
+                        uint32_t val = gpu_read(gpu, addr);
+                        printk("IO:R %08x %08x\n", addr<<2, val);
+                }
+        }
+}
+
 static const struct adreno_gpu_funcs funcs = {
         .base = {
                 .get_param = adreno_get_param,
@@ -439,7 +470,8 @@ struct msm_gpu *a3xx_gpu_init(struct drm_device *dev)
         struct a3xx_gpu *a3xx_gpu = NULL;
         struct adreno_gpu *adreno_gpu;
         struct msm_gpu *gpu;
-        struct platform_device *pdev = a3xx_pdev;
+        struct msm_drm_private *priv = dev->dev_private;
+        struct platform_device *pdev = priv->gpu_pdev;
         struct adreno_platform_config *config;
         int ret;
 
@@ -460,7 +492,6 @@ struct msm_gpu *a3xx_gpu_init(struct drm_device *dev)
         adreno_gpu = &a3xx_gpu->base;
         gpu = &adreno_gpu->base;
 
-        get_device(&pdev->dev);
         a3xx_gpu->pdev = pdev;
 
         gpu->fast_rate = config->fast_rate;
@@ -522,17 +553,24 @@ fail:
 #  include <mach/kgsl.h>
 #endif
 
-static int a3xx_probe(struct platform_device *pdev)
+static void set_gpu_pdev(struct drm_device *dev,
+                struct platform_device *pdev)
+{
+        struct msm_drm_private *priv = dev->dev_private;
+        priv->gpu_pdev = pdev;
+}
+
+static int a3xx_bind(struct device *dev, struct device *master, void *data)
 {
         static struct adreno_platform_config config = {};
 #ifdef CONFIG_OF
-        struct device_node *child, *node = pdev->dev.of_node;
+        struct device_node *child, *node = dev->of_node;
         u32 val;
         int ret;
 
         ret = of_property_read_u32(node, "qcom,chipid", &val);
         if (ret) {
-                dev_err(&pdev->dev, "could not find chipid: %d\n", ret);
+                dev_err(dev, "could not find chipid: %d\n", ret);
                 return ret;
         }
 
@@ -548,7 +586,7 @@ static int a3xx_probe(struct platform_device *pdev)
                         for_each_child_of_node(child, pwrlvl) {
                                 ret = of_property_read_u32(pwrlvl, "qcom,gpu-freq", &val);
                                 if (ret) {
-                                        dev_err(&pdev->dev, "could not find gpu-freq: %d\n", ret);
+                                        dev_err(dev, "could not find gpu-freq: %d\n", ret);
                                         return ret;
                                 }
                                 config.fast_rate = max(config.fast_rate, val);
@@ -558,12 +596,12 @@ static int a3xx_probe(struct platform_device *pdev)
         }
 
         if (!config.fast_rate) {
-                dev_err(&pdev->dev, "could not find clk rates\n");
+                dev_err(dev, "could not find clk rates\n");
                 return -ENXIO;
         }
 
 #else
-        struct kgsl_device_platform_data *pdata = pdev->dev.platform_data;
+        struct kgsl_device_platform_data *pdata = dev->platform_data;
         uint32_t version = socinfo_get_version();
         if (cpu_is_apq8064ab()) {
                 config.fast_rate = 450000000;
@@ -609,14 +647,30 @@ static int a3xx_probe(struct platform_device *pdev)
         config.bus_scale_table = pdata->bus_scale_table;
 #  endif
 #endif
-        pdev->dev.platform_data = &config;
-        a3xx_pdev = pdev;
+        dev->platform_data = &config;
+        set_gpu_pdev(dev_get_drvdata(master), to_platform_device(dev));
         return 0;
 }
 
+static void a3xx_unbind(struct device *dev, struct device *master,
+                void *data)
+{
+        set_gpu_pdev(dev_get_drvdata(master), NULL);
+}
+
+static const struct component_ops a3xx_ops = {
+                .bind   = a3xx_bind,
+                .unbind = a3xx_unbind,
+};
+
+static int a3xx_probe(struct platform_device *pdev)
+{
+        return component_add(&pdev->dev, &a3xx_ops);
+}
+
 static int a3xx_remove(struct platform_device *pdev)
 {
-        a3xx_pdev = NULL;
+        component_del(&pdev->dev, &a3xx_ops);
         return 0;
 }
 
@@ -624,7 +678,6 @@ static const struct of_device_id dt_match[] = {
         { .compatible = "qcom,kgsl-3d0" },
         {}
 };
-MODULE_DEVICE_TABLE(of, dt_match);
 
 static struct platform_driver a3xx_driver = {
         .probe = a3xx_probe,
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
index d321099abdd4..28ca8cd8b09e 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
@@ -73,6 +73,12 @@ int adreno_get_param(struct msm_gpu *gpu, uint32_t param, uint64_t *value)
         case MSM_PARAM_GMEM_SIZE:
                 *value = adreno_gpu->gmem;
                 return 0;
+        case MSM_PARAM_CHIP_ID:
+                *value = adreno_gpu->rev.patchid |
+                                (adreno_gpu->rev.minor << 8) |
+                                (adreno_gpu->rev.major << 16) |
+                                (adreno_gpu->rev.core << 24);
+                return 0;
         default:
                 DBG("%s: invalid param: %u", gpu->name, param);
                 return -EINVAL;
@@ -225,19 +231,11 @@ void adreno_flush(struct msm_gpu *gpu)
 void adreno_idle(struct msm_gpu *gpu)
 {
         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
-        uint32_t rptr, wptr = get_wptr(gpu->rb);
-        unsigned long t;
-
-        t = jiffies + ADRENO_IDLE_TIMEOUT;
-
-        /* then wait for CP to drain ringbuffer: */
-        do {
-                rptr = adreno_gpu->memptrs->rptr;
-                if (rptr == wptr)
-                        return;
-        } while(time_before(jiffies, t));
+        uint32_t wptr = get_wptr(gpu->rb);
 
-        DRM_ERROR("%s: timeout waiting to drain ringbuffer!\n", gpu->name);
+        /* wait for CP to drain ringbuffer: */
+        if (spin_until(adreno_gpu->memptrs->rptr == wptr))
+                DRM_ERROR("%s: timeout waiting to drain ringbuffer!\n", gpu->name);
 
         /* TODO maybe we need to reset GPU here to recover from hang? */
 }
@@ -260,22 +258,37 @@ void adreno_show(struct msm_gpu *gpu, struct seq_file *m)
 }
 #endif
 
-void adreno_wait_ring(struct msm_gpu *gpu, uint32_t ndwords)
+/* would be nice to not have to duplicate the _show() stuff with printk(): */
+void adreno_dump(struct msm_gpu *gpu)
 {
         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
-        uint32_t freedwords;
-        unsigned long t = jiffies + ADRENO_IDLE_TIMEOUT;
-        do {
-                uint32_t size = gpu->rb->size / 4;
-                uint32_t wptr = get_wptr(gpu->rb);
-                uint32_t rptr = adreno_gpu->memptrs->rptr;
-                freedwords = (rptr + (size - 1) - wptr) % size;
-
-                if (time_after(jiffies, t)) {
-                        DRM_ERROR("%s: timeout waiting for ringbuffer space\n", gpu->name);
-                        break;
-                }
-        } while(freedwords < ndwords);
+
+        printk("revision: %d (%d.%d.%d.%d)\n",
+                        adreno_gpu->info->revn, adreno_gpu->rev.core,
+                        adreno_gpu->rev.major, adreno_gpu->rev.minor,
+                        adreno_gpu->rev.patchid);
+
+        printk("fence:    %d/%d\n", adreno_gpu->memptrs->fence,
+                        gpu->submitted_fence);
+        printk("rptr:     %d\n", adreno_gpu->memptrs->rptr);
+        printk("wptr:     %d\n", adreno_gpu->memptrs->wptr);
+        printk("rb wptr:  %d\n", get_wptr(gpu->rb));
+
+}
+
+static uint32_t ring_freewords(struct msm_gpu *gpu)
+{
+        struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
+        uint32_t size = gpu->rb->size / 4;
+        uint32_t wptr = get_wptr(gpu->rb);
+        uint32_t rptr = adreno_gpu->memptrs->rptr;
+        return (rptr + (size - 1) - wptr) % size;
+}
+
+void adreno_wait_ring(struct msm_gpu *gpu, uint32_t ndwords)
+{
+        if (spin_until(ring_freewords(gpu) >= ndwords))
+                DRM_ERROR("%s: timeout waiting for ringbuffer space\n", gpu->name);
 }
 
 static const char *iommu_ports[] = {
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
index ca11ea4da165..63c36ce33020 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h
@@ -76,7 +76,20 @@ struct adreno_platform_config {
 #endif
 };
 
-#define ADRENO_IDLE_TIMEOUT (20 * 1000)
+#define ADRENO_IDLE_TIMEOUT msecs_to_jiffies(1000)
+
+#define spin_until(X) ({                                   \
+        int __ret = -ETIMEDOUT;                            \
+        unsigned long __t = jiffies + ADRENO_IDLE_TIMEOUT; \
+        do {                                               \
+                if (X) {                                   \
+                        __ret = 0;                         \
+                        break;                             \
+                }                                          \
+        } while (time_before(jiffies, __t));               \
+        __ret;                                             \
+})
+
 
 static inline bool adreno_is_a3xx(struct adreno_gpu *gpu)
 {
@@ -114,6 +127,7 @@ void adreno_idle(struct msm_gpu *gpu);
 #ifdef CONFIG_DEBUG_FS
 void adreno_show(struct msm_gpu *gpu, struct seq_file *m);
 #endif
+void adreno_dump(struct msm_gpu *gpu);
 void adreno_wait_ring(struct msm_gpu *gpu, uint32_t ndwords);
 
 int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.c b/drivers/gpu/drm/msm/hdmi/hdmi.c
index 6f1588aa9071..ae750f6928c1 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi.c
@@ -17,8 +17,6 @@
 
 #include "hdmi.h"
 
-static struct platform_device *hdmi_pdev;
-
 void hdmi_set_mode(struct hdmi *hdmi, bool power_on)
 {
         uint32_t ctrl = 0;
@@ -67,7 +65,7 @@ void hdmi_destroy(struct kref *kref)
         if (hdmi->i2c)
                 hdmi_i2c_destroy(hdmi->i2c);
 
-        put_device(&hdmi->pdev->dev);
+        platform_set_drvdata(hdmi->pdev, NULL);
 }
 
 /* initialize connector */
@@ -75,7 +73,7 @@ struct hdmi *hdmi_init(struct drm_device *dev, struct drm_encoder *encoder)
 {
         struct hdmi *hdmi = NULL;
         struct msm_drm_private *priv = dev->dev_private;
-        struct platform_device *pdev = hdmi_pdev;
+        struct platform_device *pdev = priv->hdmi_pdev;
         struct hdmi_platform_config *config;
         int i, ret;
 
@@ -95,13 +93,13 @@ struct hdmi *hdmi_init(struct drm_device *dev, struct drm_encoder *encoder)
 
         kref_init(&hdmi->refcount);
 
-        get_device(&pdev->dev);
-
         hdmi->dev = dev;
         hdmi->pdev = pdev;
         hdmi->config = config;
         hdmi->encoder = encoder;
 
+        hdmi_audio_infoframe_init(&hdmi->audio.infoframe);
+
         /* not sure about which phy maps to which msm.. probably I miss some */
         if (config->phy_init)
                 hdmi->phy = config->phy_init(hdmi);
@@ -228,6 +226,8 @@ struct hdmi *hdmi_init(struct drm_device *dev, struct drm_encoder *encoder)
         priv->bridges[priv->num_bridges++]       = hdmi->bridge;
         priv->connectors[priv->num_connectors++] = hdmi->connector;
 
+        platform_set_drvdata(pdev, hdmi);
+
         return hdmi;
 
 fail:
@@ -249,17 +249,24 @@ fail:
 
 #include <linux/of_gpio.h>
 
-static int hdmi_dev_probe(struct platform_device *pdev)
+static void set_hdmi_pdev(struct drm_device *dev,
+                struct platform_device *pdev)
+{
+        struct msm_drm_private *priv = dev->dev_private;
+        priv->hdmi_pdev = pdev;
+}
+
+static int hdmi_bind(struct device *dev, struct device *master, void *data)
 {
         static struct hdmi_platform_config config = {};
 #ifdef CONFIG_OF
-        struct device_node *of_node = pdev->dev.of_node;
+        struct device_node *of_node = dev->of_node;
 
         int get_gpio(const char *name)
         {
                 int gpio = of_get_named_gpio(of_node, name, 0);
                 if (gpio < 0) {
-                        dev_err(&pdev->dev, "failed to get gpio: %s (%d)\n",
+                        dev_err(dev, "failed to get gpio: %s (%d)\n",
                                         name, gpio);
                         gpio = -1;
                 }
@@ -305,7 +312,7 @@ static int hdmi_dev_probe(struct platform_device *pdev)
                 config.ddc_data_gpio = 71;
                 config.hpd_gpio      = 72;
                 config.mux_en_gpio   = -1;
-                config.mux_sel_gpio  = 13 + NR_GPIO_IRQS;
+                config.mux_sel_gpio  = -1;
         } else if (cpu_is_msm8960() || cpu_is_msm8960ab()) {
                 static const char *hpd_reg_names[] = {"8921_hdmi_mvs"};
                 config.phy_init      = hdmi_phy_8960_init;
@@ -336,14 +343,30 @@ static int hdmi_dev_probe(struct platform_device *pdev)
                 config.mux_sel_gpio  = -1;
         }
 #endif
-        pdev->dev.platform_data = &config;
-        hdmi_pdev = pdev;
+        dev->platform_data = &config;
+        set_hdmi_pdev(dev_get_drvdata(master), to_platform_device(dev));
         return 0;
 }
 
+static void hdmi_unbind(struct device *dev, struct device *master,
+                void *data)
+{
+        set_hdmi_pdev(dev_get_drvdata(master), NULL);
+}
+
+static const struct component_ops hdmi_ops = {
+                .bind   = hdmi_bind,
+                .unbind = hdmi_unbind,
+};
+
+static int hdmi_dev_probe(struct platform_device *pdev)
+{
+        return component_add(&pdev->dev, &hdmi_ops);
+}
+
 static int hdmi_dev_remove(struct platform_device *pdev)
 {
-        hdmi_pdev = NULL;
+        component_del(&pdev->dev, &hdmi_ops);
         return 0;
 }
 
@@ -351,7 +374,6 @@ static const struct of_device_id dt_match[] = {
         { .compatible = "qcom,hdmi-tx" },
         {}
 };
-MODULE_DEVICE_TABLE(of, dt_match);
 
 static struct platform_driver hdmi_driver = {
         .probe = hdmi_dev_probe,
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.h b/drivers/gpu/drm/msm/hdmi/hdmi.h
index 41b29add70b1..9fafee6a3e43 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi.h
+++ b/drivers/gpu/drm/msm/hdmi/hdmi.h
@@ -22,6 +22,7 @@
 #include <linux/clk.h>
 #include <linux/platform_device.h>
 #include <linux/regulator/consumer.h>
+#include <linux/hdmi.h>
 
 #include "msm_drv.h"
 #include "hdmi.xml.h"
@@ -30,6 +31,12 @@
 struct hdmi_phy;
 struct hdmi_platform_config;
 
+struct hdmi_audio {
+        bool enabled;
+        struct hdmi_audio_infoframe infoframe;
+        int rate;
+};
+
 struct hdmi {
         struct kref refcount;
 
@@ -38,6 +45,13 @@ struct hdmi {
 
         const struct hdmi_platform_config *config;
 
+        /* audio state: */
+        struct hdmi_audio audio;
+
+        /* video state: */
+        bool power_on;
+        unsigned long int pixclock;
+
         void __iomem *mmio;
 
         struct regulator *hpd_regs[2];
@@ -132,6 +146,17 @@ struct hdmi_phy *hdmi_phy_8x60_init(struct hdmi *hdmi);
 struct hdmi_phy *hdmi_phy_8x74_init(struct hdmi *hdmi);
 
 /*
+ * audio:
+ */
+
+int hdmi_audio_update(struct hdmi *hdmi);
+int hdmi_audio_info_setup(struct hdmi *hdmi, bool enabled,
+        uint32_t num_of_channels, uint32_t channel_allocation,
+        uint32_t level_shift, bool down_mix);
+void hdmi_audio_set_sample_rate(struct hdmi *hdmi, int rate);
+
+
+/*
  * hdmi bridge:
  */
 
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_audio.c b/drivers/gpu/drm/msm/hdmi/hdmi_audio.c
new file mode 100644
index 000000000000..872485f60134
--- /dev/null
+++ b/drivers/gpu/drm/msm/hdmi/hdmi_audio.c
@@ -0,0 +1,273 @@
+/*
+ * Copyright (C) 2013 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/hdmi.h>
+#include "hdmi.h"
+
+
+/* Supported HDMI Audio channels */
+#define MSM_HDMI_AUDIO_CHANNEL_2                0
+#define MSM_HDMI_AUDIO_CHANNEL_4                1
+#define MSM_HDMI_AUDIO_CHANNEL_6                2
+#define MSM_HDMI_AUDIO_CHANNEL_8                3
+
+/* maps MSM_HDMI_AUDIO_CHANNEL_n consts used by audio driver to # of channels: */
+static int nchannels[] = { 2, 4, 6, 8 };
+
+/* Supported HDMI Audio sample rates */
+#define MSM_HDMI_SAMPLE_RATE_32KHZ              0
+#define MSM_HDMI_SAMPLE_RATE_44_1KHZ            1
+#define MSM_HDMI_SAMPLE_RATE_48KHZ              2
+#define MSM_HDMI_SAMPLE_RATE_88_2KHZ            3
+#define MSM_HDMI_SAMPLE_RATE_96KHZ              4
+#define MSM_HDMI_SAMPLE_RATE_176_4KHZ           5
+#define MSM_HDMI_SAMPLE_RATE_192KHZ             6
+#define MSM_HDMI_SAMPLE_RATE_MAX                7
+
+
+struct hdmi_msm_audio_acr {
+        uint32_t n;     /* N parameter for clock regeneration */
+        uint32_t cts;   /* CTS parameter for clock regeneration */
+};
+
+struct hdmi_msm_audio_arcs {
+        unsigned long int pixclock;
+        struct hdmi_msm_audio_acr lut[MSM_HDMI_SAMPLE_RATE_MAX];
+};
+
+#define HDMI_MSM_AUDIO_ARCS(pclk, ...) { (1000 * (pclk)), __VA_ARGS__ }
+
+/* Audio constants lookup table for hdmi_msm_audio_acr_setup */
+/* Valid Pixel-Clock rates: 25.2MHz, 27MHz, 27.03MHz, 74.25MHz, 148.5MHz */
+static const struct hdmi_msm_audio_arcs acr_lut[] = {
+        /*  25.200MHz  */
+        HDMI_MSM_AUDIO_ARCS(25200, {
+                {4096, 25200}, {6272, 28000}, {6144, 25200}, {12544, 28000},
+                {12288, 25200}, {25088, 28000}, {24576, 25200} }),
+        /*  27.000MHz  */
+        HDMI_MSM_AUDIO_ARCS(27000, {
+                {4096, 27000}, {6272, 30000}, {6144, 27000}, {12544, 30000},
+                {12288, 27000}, {25088, 30000}, {24576, 27000} }),
+        /*  27.027MHz */
+        HDMI_MSM_AUDIO_ARCS(27030, {
+                {4096, 27027}, {6272, 30030}, {6144, 27027}, {12544, 30030},
+                {12288, 27027}, {25088, 30030}, {24576, 27027} }),
+        /*  74.250MHz */
+        HDMI_MSM_AUDIO_ARCS(74250, {
+                {4096, 74250}, {6272, 82500}, {6144, 74250}, {12544, 82500},
+                {12288, 74250}, {25088, 82500}, {24576, 74250} }),
+        /* 148.500MHz */
+        HDMI_MSM_AUDIO_ARCS(148500, {
+                {4096, 148500}, {6272, 165000}, {6144, 148500}, {12544, 165000},
+                {12288, 148500}, {25088, 165000}, {24576, 148500} }),
+};
+
+static const struct hdmi_msm_audio_arcs *get_arcs(unsigned long int pixclock)
+{
+        int i;
+
+        for (i = 0; i < ARRAY_SIZE(acr_lut); i++) {
+                const struct hdmi_msm_audio_arcs *arcs = &acr_lut[i];
+                if (arcs->pixclock == pixclock)
+                        return arcs;
+        }
+
+        return NULL;
+}
+
+int hdmi_audio_update(struct hdmi *hdmi)
+{
+        struct hdmi_audio *audio = &hdmi->audio;
+        struct hdmi_audio_infoframe *info = &audio->infoframe;
+        const struct hdmi_msm_audio_arcs *arcs = NULL;
+        bool enabled = audio->enabled;
+        uint32_t acr_pkt_ctrl, vbi_pkt_ctrl, aud_pkt_ctrl;
+        uint32_t infofrm_ctrl, audio_config;
+
+        DBG("audio: enabled=%d, channels=%d, channel_allocation=0x%x, "
+                "level_shift_value=%d, downmix_inhibit=%d, rate=%d",
+                audio->enabled, info->channels,  info->channel_allocation,
+                info->level_shift_value, info->downmix_inhibit, audio->rate);
+        DBG("video: power_on=%d, pixclock=%lu", hdmi->power_on, hdmi->pixclock);
+
+        if (enabled && !(hdmi->power_on && hdmi->pixclock)) {
+                DBG("disabling audio: no video");
+                enabled = false;
+        }
+
+        if (enabled) {
+                arcs = get_arcs(hdmi->pixclock);
+                if (!arcs) {
+                        DBG("disabling audio: unsupported pixclock: %lu",
+                                        hdmi->pixclock);
+                        enabled = false;
+                }
+        }
+
+        /* Read first before writing */
+        acr_pkt_ctrl = hdmi_read(hdmi, REG_HDMI_ACR_PKT_CTRL);
+        vbi_pkt_ctrl = hdmi_read(hdmi, REG_HDMI_VBI_PKT_CTRL);
+        aud_pkt_ctrl = hdmi_read(hdmi, REG_HDMI_AUDIO_PKT_CTRL1);
+        infofrm_ctrl = hdmi_read(hdmi, REG_HDMI_INFOFRAME_CTRL0);
+        audio_config = hdmi_read(hdmi, REG_HDMI_AUDIO_CFG);
+
+        /* Clear N/CTS selection bits */
+        acr_pkt_ctrl &= ~HDMI_ACR_PKT_CTRL_SELECT__MASK;
+
+        if (enabled) {
+                uint32_t n, cts, multiplier;
+                enum hdmi_acr_cts select;
+                uint8_t buf[14];
+
+                n   = arcs->lut[audio->rate].n;
+                cts = arcs->lut[audio->rate].cts;
+
+                if ((MSM_HDMI_SAMPLE_RATE_192KHZ == audio->rate) ||
+                                (MSM_HDMI_SAMPLE_RATE_176_4KHZ == audio->rate)) {
+                        multiplier = 4;
+                        n >>= 2; /* divide N by 4 and use multiplier */
+                } else if ((MSM_HDMI_SAMPLE_RATE_96KHZ == audio->rate) ||
+                                (MSM_HDMI_SAMPLE_RATE_88_2KHZ == audio->rate)) {
+                        multiplier = 2;
+                        n >>= 1; /* divide N by 2 and use multiplier */
+                } else {
+                        multiplier = 1;
+                }
+
+                DBG("n=%u, cts=%u, multiplier=%u", n, cts, multiplier);
+
+                acr_pkt_ctrl |= HDMI_ACR_PKT_CTRL_SOURCE;
+                acr_pkt_ctrl |= HDMI_ACR_PKT_CTRL_AUDIO_PRIORITY;
+                acr_pkt_ctrl |= HDMI_ACR_PKT_CTRL_N_MULTIPLIER(multiplier);
+
+                if ((MSM_HDMI_SAMPLE_RATE_48KHZ == audio->rate) ||
+                                (MSM_HDMI_SAMPLE_RATE_96KHZ == audio->rate) ||
+                                (MSM_HDMI_SAMPLE_RATE_192KHZ == audio->rate))
+                        select = ACR_48;
+                else if ((MSM_HDMI_SAMPLE_RATE_44_1KHZ == audio->rate) ||
+                                (MSM_HDMI_SAMPLE_RATE_88_2KHZ == audio->rate) ||
+                                (MSM_HDMI_SAMPLE_RATE_176_4KHZ == audio->rate))
+                        select = ACR_44;
+                else /* default to 32k */
+                        select = ACR_32;
+
+                acr_pkt_ctrl |= HDMI_ACR_PKT_CTRL_SELECT(select);
+
+                hdmi_write(hdmi, REG_HDMI_ACR_0(select - 1),
+                                HDMI_ACR_0_CTS(cts));
+                hdmi_write(hdmi, REG_HDMI_ACR_1(select - 1),
+                                HDMI_ACR_1_N(n));
+
+                hdmi_write(hdmi, REG_HDMI_AUDIO_PKT_CTRL2,
+                                COND(info->channels != 2, HDMI_AUDIO_PKT_CTRL2_LAYOUT) |
+                                HDMI_AUDIO_PKT_CTRL2_OVERRIDE);
+
+                acr_pkt_ctrl |= HDMI_ACR_PKT_CTRL_CONT;
+                acr_pkt_ctrl |= HDMI_ACR_PKT_CTRL_SEND;
+
+                /* configure infoframe: */
+                hdmi_audio_infoframe_pack(info, buf, sizeof(buf));
+                hdmi_write(hdmi, REG_HDMI_AUDIO_INFO0,
+                                (buf[3] <<  0) || (buf[4] <<  8) ||
+                                (buf[5] << 16) || (buf[6] << 24));
+                hdmi_write(hdmi, REG_HDMI_AUDIO_INFO1,
+                                (buf[7] <<  0) || (buf[8] << 8));
+
+                hdmi_write(hdmi, REG_HDMI_GC, 0);
+
+                vbi_pkt_ctrl |= HDMI_VBI_PKT_CTRL_GC_ENABLE;
+                vbi_pkt_ctrl |= HDMI_VBI_PKT_CTRL_GC_EVERY_FRAME;
+
+                aud_pkt_ctrl |= HDMI_AUDIO_PKT_CTRL1_AUDIO_SAMPLE_SEND;
+
+                infofrm_ctrl |= HDMI_INFOFRAME_CTRL0_AUDIO_INFO_SEND;
+                infofrm_ctrl |= HDMI_INFOFRAME_CTRL0_AUDIO_INFO_CONT;
+                infofrm_ctrl |= HDMI_INFOFRAME_CTRL0_AUDIO_INFO_SOURCE;
+                infofrm_ctrl |= HDMI_INFOFRAME_CTRL0_AUDIO_INFO_UPDATE;
+
+                audio_config &= ~HDMI_AUDIO_CFG_FIFO_WATERMARK__MASK;
+                audio_config |= HDMI_AUDIO_CFG_FIFO_WATERMARK(4);
+                audio_config |= HDMI_AUDIO_CFG_ENGINE_ENABLE;
+        } else {
+                hdmi_write(hdmi, REG_HDMI_GC, HDMI_GC_MUTE);
+                acr_pkt_ctrl &= ~HDMI_ACR_PKT_CTRL_CONT;
+                acr_pkt_ctrl &= ~HDMI_ACR_PKT_CTRL_SEND;
+                vbi_pkt_ctrl &= ~HDMI_VBI_PKT_CTRL_GC_ENABLE;
+                vbi_pkt_ctrl &= ~HDMI_VBI_PKT_CTRL_GC_EVERY_FRAME;
+                aud_pkt_ctrl &= ~HDMI_AUDIO_PKT_CTRL1_AUDIO_SAMPLE_SEND;
+                infofrm_ctrl &= ~HDMI_INFOFRAME_CTRL0_AUDIO_INFO_SEND;
+                infofrm_ctrl &= ~HDMI_INFOFRAME_CTRL0_AUDIO_INFO_CONT;
+                infofrm_ctrl &= ~HDMI_INFOFRAME_CTRL0_AUDIO_INFO_SOURCE;
+                infofrm_ctrl &= ~HDMI_INFOFRAME_CTRL0_AUDIO_INFO_UPDATE;
+                audio_config &= ~HDMI_AUDIO_CFG_ENGINE_ENABLE;
+        }
+
+        hdmi_write(hdmi, REG_HDMI_ACR_PKT_CTRL, acr_pkt_ctrl);
+        hdmi_write(hdmi, REG_HDMI_VBI_PKT_CTRL, vbi_pkt_ctrl);
+        hdmi_write(hdmi, REG_HDMI_AUDIO_PKT_CTRL1, aud_pkt_ctrl);
+        hdmi_write(hdmi, REG_HDMI_INFOFRAME_CTRL0, infofrm_ctrl);
+
+        hdmi_write(hdmi, REG_HDMI_AUD_INT,
+                        COND(enabled, HDMI_AUD_INT_AUD_FIFO_URUN_INT) |
+                        COND(enabled, HDMI_AUD_INT_AUD_SAM_DROP_INT));
+
+        hdmi_write(hdmi, REG_HDMI_AUDIO_CFG, audio_config);
+
+
+        DBG("audio %sabled", enabled ? "en" : "dis");
+
+        return 0;
+}
+
+int hdmi_audio_info_setup(struct hdmi *hdmi, bool enabled,
+        uint32_t num_of_channels, uint32_t channel_allocation,
+        uint32_t level_shift, bool down_mix)
+{
+        struct hdmi_audio *audio;
+
+        if (!hdmi)
+                return -ENXIO;
+
+        audio = &hdmi->audio;
+
+        if (num_of_channels >= ARRAY_SIZE(nchannels))
+                return -EINVAL;
+
+        audio->enabled = enabled;
+        audio->infoframe.channels = nchannels[num_of_channels];
+        audio->infoframe.channel_allocation = channel_allocation;
+        audio->infoframe.level_shift_value = level_shift;
+        audio->infoframe.downmix_inhibit = down_mix;
+
+        return hdmi_audio_update(hdmi);
+}
+
+void hdmi_audio_set_sample_rate(struct hdmi *hdmi, int rate)
+{
+        struct hdmi_audio *audio;
+
+        if (!hdmi)
+                return;
+
+        audio = &hdmi->audio;
+
+        if ((rate < 0) || (rate >= MSM_HDMI_SAMPLE_RATE_MAX))
+                return;
+
+        audio->rate = rate;
+        hdmi_audio_update(hdmi);
+}
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c b/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c
index 7d10e55403c6..f6cf745c249e 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi_bridge.c
@@ -19,11 +19,7 @@
 
 struct hdmi_bridge {
         struct drm_bridge base;
-
         struct hdmi *hdmi;
-        bool power_on;
-
-        unsigned long int pixclock;
 };
 #define to_hdmi_bridge(x) container_of(x, struct hdmi_bridge, base)
 
@@ -52,8 +48,8 @@ static void power_on(struct drm_bridge *bridge)
         }
 
         if (config->pwr_clk_cnt > 0) {
-                DBG("pixclock: %lu", hdmi_bridge->pixclock);
-                ret = clk_set_rate(hdmi->pwr_clks[0], hdmi_bridge->pixclock);
+                DBG("pixclock: %lu", hdmi->pixclock);
+                ret = clk_set_rate(hdmi->pwr_clks[0], hdmi->pixclock);
                 if (ret) {
                         dev_err(dev->dev, "failed to set pixel clk: %s (%d)\n",
                                         config->pwr_clk_names[0], ret);
@@ -102,12 +98,13 @@ static void hdmi_bridge_pre_enable(struct drm_bridge *bridge)
 
         DBG("power up");
 
-        if (!hdmi_bridge->power_on) {
+        if (!hdmi->power_on) {
                 power_on(bridge);
-                hdmi_bridge->power_on = true;
+                hdmi->power_on = true;
+                hdmi_audio_update(hdmi);
         }
 
-        phy->funcs->powerup(phy, hdmi_bridge->pixclock);
+        phy->funcs->powerup(phy, hdmi->pixclock);
         hdmi_set_mode(hdmi, true);
 }
 
@@ -129,9 +126,10 @@ static void hdmi_bridge_post_disable(struct drm_bridge *bridge)
         hdmi_set_mode(hdmi, false);
         phy->funcs->powerdown(phy);
 
-        if (hdmi_bridge->power_on) {
+        if (hdmi->power_on) {
                 power_off(bridge);
-                hdmi_bridge->power_on = false;
+                hdmi->power_on = false;
+                hdmi_audio_update(hdmi);
         }
 }
 
@@ -146,7 +144,7 @@ static void hdmi_bridge_mode_set(struct drm_bridge *bridge,
 
         mode = adjusted_mode;
 
-        hdmi_bridge->pixclock = mode->clock * 1000;
+        hdmi->pixclock = mode->clock * 1000;
 
         hdmi->hdmi_mode = drm_match_cea_mode(mode) > 1;
 
@@ -194,9 +192,7 @@ static void hdmi_bridge_mode_set(struct drm_bridge *bridge,
         DBG("frame_ctrl=%08x", frame_ctrl);
         hdmi_write(hdmi, REG_HDMI_FRAME_CTRL, frame_ctrl);
 
-        // TODO until we have audio, this might be safest:
-        if (hdmi->hdmi_mode)
-                hdmi_write(hdmi, REG_HDMI_GC, HDMI_GC_MUTE);
+        hdmi_audio_update(hdmi);
 }
 
 static const struct drm_bridge_funcs hdmi_bridge_funcs = {
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c
index 84c5b13b33c9..3e6c0f3ed592 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_crtc.c
@@ -120,7 +120,7 @@ static void update_fb(struct drm_crtc *crtc, struct drm_framebuffer *new_fb)
 
         /* grab reference to incoming scanout fb: */
         drm_framebuffer_reference(new_fb);
-        mdp4_crtc->base.fb = new_fb;
+        mdp4_crtc->base.primary->fb = new_fb;
         mdp4_crtc->fb = new_fb;
 
         if (old_fb)
@@ -182,7 +182,7 @@ static void pageflip_cb(struct msm_fence_cb *cb)
         struct mdp4_crtc *mdp4_crtc =
                 container_of(cb, struct mdp4_crtc, pageflip_cb);
         struct drm_crtc *crtc = &mdp4_crtc->base;
-        struct drm_framebuffer *fb = crtc->fb;
+        struct drm_framebuffer *fb = crtc->primary->fb;
 
         if (!fb)
                 return;
@@ -348,14 +348,14 @@ static int mdp4_crtc_mode_set(struct drm_crtc *crtc,
                         mode->type, mode->flags);
 
         /* grab extra ref for update_scanout() */
-        drm_framebuffer_reference(crtc->fb);
+        drm_framebuffer_reference(crtc->primary->fb);
 
-        ret = mdp4_plane_mode_set(mdp4_crtc->plane, crtc, crtc->fb,
+        ret = mdp4_plane_mode_set(mdp4_crtc->plane, crtc, crtc->primary->fb,
                         0, 0, mode->hdisplay, mode->vdisplay,
                         x << 16, y << 16,
                         mode->hdisplay << 16, mode->vdisplay << 16);
         if (ret) {
-                drm_framebuffer_unreference(crtc->fb);
+                drm_framebuffer_unreference(crtc->primary->fb);
                 dev_err(crtc->dev->dev, "%s: failed to set mode on plane: %d\n",
                                 mdp4_crtc->name, ret);
                 return ret;
@@ -368,7 +368,7 @@ static int mdp4_crtc_mode_set(struct drm_crtc *crtc,
         /* take data from pipe: */
         mdp4_write(mdp4_kms, REG_MDP4_DMA_SRC_BASE(dma), 0);
         mdp4_write(mdp4_kms, REG_MDP4_DMA_SRC_STRIDE(dma),
-                        crtc->fb->pitches[0]);
+                        crtc->primary->fb->pitches[0]);
         mdp4_write(mdp4_kms, REG_MDP4_DMA_DST_SIZE(dma),
                         MDP4_DMA_DST_SIZE_WIDTH(0) |
                         MDP4_DMA_DST_SIZE_HEIGHT(0));
@@ -378,7 +378,7 @@ static int mdp4_crtc_mode_set(struct drm_crtc *crtc,
                         MDP4_OVLP_SIZE_WIDTH(mode->hdisplay) |
                         MDP4_OVLP_SIZE_HEIGHT(mode->vdisplay));
         mdp4_write(mdp4_kms, REG_MDP4_OVLP_STRIDE(ovlp),
-                        crtc->fb->pitches[0]);
+                        crtc->primary->fb->pitches[0]);
 
         mdp4_write(mdp4_kms, REG_MDP4_OVLP_CFG(ovlp), 1);
 
@@ -388,8 +388,8 @@ static int mdp4_crtc_mode_set(struct drm_crtc *crtc,
                 mdp4_write(mdp4_kms, REG_MDP4_DMA_E_QUANT(2), 0x00ff0000);
         }
 
-        update_fb(crtc, crtc->fb);
-        update_scanout(crtc, crtc->fb);
+        update_fb(crtc, crtc->primary->fb);
+        update_scanout(crtc, crtc->primary->fb);
 
         return 0;
 }
@@ -420,19 +420,19 @@ static int mdp4_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
         int ret;
 
         /* grab extra ref for update_scanout() */
-        drm_framebuffer_reference(crtc->fb);
+        drm_framebuffer_reference(crtc->primary->fb);
 
-        ret = mdp4_plane_mode_set(plane, crtc, crtc->fb,
+        ret = mdp4_plane_mode_set(plane, crtc, crtc->primary->fb,
                         0, 0, mode->hdisplay, mode->vdisplay,
                         x << 16, y << 16,
                         mode->hdisplay << 16, mode->vdisplay << 16);
         if (ret) {
-                drm_framebuffer_unreference(crtc->fb);
+                drm_framebuffer_unreference(crtc->primary->fb);
                 return ret;
         }
 
-        update_fb(crtc, crtc->fb);
-        update_scanout(crtc, crtc->fb);
+        update_fb(crtc, crtc->primary->fb);
+        update_scanout(crtc, crtc->primary->fb);
 
         return 0;
 }
@@ -740,6 +740,9 @@ void mdp4_crtc_attach(struct drm_crtc *crtc, struct drm_plane *plane)
 
 void mdp4_crtc_detach(struct drm_crtc *crtc, struct drm_plane *plane)
 {
+        /* don't actually detatch our primary plane: */
+        if (to_mdp4_crtc(crtc)->plane == plane)
+                return;
         set_attach(crtc, mdp4_plane_pipe(plane), NULL);
 }
 
@@ -791,7 +794,7 @@ struct drm_crtc *mdp4_crtc_init(struct drm_device *dev,
 
         INIT_FENCE_CB(&mdp4_crtc->pageflip_cb, pageflip_cb);
 
-        drm_crtc_init(dev, crtc, &mdp4_crtc_funcs);
+        drm_crtc_init_with_planes(dev, crtc, plane, NULL, &mdp4_crtc_funcs);
         drm_crtc_helper_add(crtc, &mdp4_crtc_helper_funcs);
 
         mdp4_plane_install_properties(mdp4_crtc->plane, &crtc->base);
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
index 1e893dd13859..66f33dba1ebb 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_plane.c
@@ -222,6 +222,7 @@ struct drm_plane *mdp4_plane_init(struct drm_device *dev,
         struct drm_plane *plane = NULL;
         struct mdp4_plane *mdp4_plane;
         int ret;
+        enum drm_plane_type type;
 
         mdp4_plane = kzalloc(sizeof(*mdp4_plane), GFP_KERNEL);
         if (!mdp4_plane) {
@@ -237,9 +238,10 @@ struct drm_plane *mdp4_plane_init(struct drm_device *dev,
         mdp4_plane->nformats = mdp4_get_formats(pipe_id, mdp4_plane->formats,
                         ARRAY_SIZE(mdp4_plane->formats));
 
-        drm_plane_init(dev, plane, 0xff, &mdp4_plane_funcs,
-                        mdp4_plane->formats, mdp4_plane->nformats,
-                        private_plane);
+        type = private_plane ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
+        drm_universal_plane_init(dev, plane, 0xff, &mdp4_plane_funcs,
+                                 mdp4_plane->formats, mdp4_plane->nformats,
+                                 type);
 
         mdp4_plane_install_properties(plane, &plane->base);
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
index f2794021f086..6ea10bdb6e8f 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
@@ -102,7 +102,7 @@ static void update_fb(struct drm_crtc *crtc, struct drm_framebuffer *new_fb)
 
         /* grab reference to incoming scanout fb: */
         drm_framebuffer_reference(new_fb);
-        mdp5_crtc->base.fb = new_fb;
+        mdp5_crtc->base.primary->fb = new_fb;
         mdp5_crtc->fb = new_fb;
 
         if (old_fb)
@@ -289,14 +289,14 @@ static int mdp5_crtc_mode_set(struct drm_crtc *crtc,
                         mode->type, mode->flags);
 
         /* grab extra ref for update_scanout() */
-        drm_framebuffer_reference(crtc->fb);
+        drm_framebuffer_reference(crtc->primary->fb);
 
-        ret = mdp5_plane_mode_set(mdp5_crtc->plane, crtc, crtc->fb,
+        ret = mdp5_plane_mode_set(mdp5_crtc->plane, crtc, crtc->primary->fb,
                         0, 0, mode->hdisplay, mode->vdisplay,
                         x << 16, y << 16,
                         mode->hdisplay << 16, mode->vdisplay << 16);
         if (ret) {
-                drm_framebuffer_unreference(crtc->fb);
+                drm_framebuffer_unreference(crtc->primary->fb);
                 dev_err(crtc->dev->dev, "%s: failed to set mode on plane: %d\n",
                                 mdp5_crtc->name, ret);
                 return ret;
@@ -306,8 +306,8 @@ static int mdp5_crtc_mode_set(struct drm_crtc *crtc,
                         MDP5_LM_OUT_SIZE_WIDTH(mode->hdisplay) |
                         MDP5_LM_OUT_SIZE_HEIGHT(mode->vdisplay));
 
-        update_fb(crtc, crtc->fb);
-        update_scanout(crtc, crtc->fb);
+        update_fb(crtc, crtc->primary->fb);
+        update_scanout(crtc, crtc->primary->fb);
 
         return 0;
 }
@@ -338,19 +338,19 @@ static int mdp5_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
         int ret;
 
         /* grab extra ref for update_scanout() */
-        drm_framebuffer_reference(crtc->fb);
+        drm_framebuffer_reference(crtc->primary->fb);
 
-        ret = mdp5_plane_mode_set(plane, crtc, crtc->fb,
+        ret = mdp5_plane_mode_set(plane, crtc, crtc->primary->fb,
                         0, 0, mode->hdisplay, mode->vdisplay,
                         x << 16, y << 16,
                         mode->hdisplay << 16, mode->vdisplay << 16);
         if (ret) {
-                drm_framebuffer_unreference(crtc->fb);
+                drm_framebuffer_unreference(crtc->primary->fb);
                 return ret;
         }
 
-        update_fb(crtc, crtc->fb);
-        update_scanout(crtc, crtc->fb);
+        update_fb(crtc, crtc->primary->fb);
+        update_scanout(crtc, crtc->primary->fb);
 
         return 0;
 }
@@ -524,6 +524,9 @@ void mdp5_crtc_attach(struct drm_crtc *crtc, struct drm_plane *plane)
 
 void mdp5_crtc_detach(struct drm_crtc *crtc, struct drm_plane *plane)
 {
+        /* don't actually detatch our primary plane: */
+        if (to_mdp5_crtc(crtc)->plane == plane)
+                return;
         set_attach(crtc, mdp5_plane_pipe(plane), NULL);
 }
 
@@ -559,7 +562,7 @@ struct drm_crtc *mdp5_crtc_init(struct drm_device *dev,
 
         INIT_FENCE_CB(&mdp5_crtc->pageflip_cb, pageflip_cb);
 
-        drm_crtc_init(dev, crtc, &mdp5_crtc_funcs);
+        drm_crtc_init_with_planes(dev, crtc, plane, NULL, &mdp5_crtc_funcs);
         drm_crtc_helper_add(crtc, &mdp5_crtc_helper_funcs);
 
         mdp5_plane_install_properties(mdp5_crtc->plane, &crtc->base);
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
index 0ac8bb5e7e85..47f7bbb9c15a 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_plane.c
@@ -358,6 +358,7 @@ struct drm_plane *mdp5_plane_init(struct drm_device *dev,
         struct drm_plane *plane = NULL;
         struct mdp5_plane *mdp5_plane;
         int ret;
+        enum drm_plane_type type;
 
         mdp5_plane = kzalloc(sizeof(*mdp5_plane), GFP_KERNEL);
         if (!mdp5_plane) {
@@ -373,9 +374,10 @@ struct drm_plane *mdp5_plane_init(struct drm_device *dev,
         mdp5_plane->nformats = mdp5_get_formats(pipe, mdp5_plane->formats,
                         ARRAY_SIZE(mdp5_plane->formats));
 
-        drm_plane_init(dev, plane, 0xff, &mdp5_plane_funcs,
-                        mdp5_plane->formats, mdp5_plane->nformats,
-                        private_plane);
+        type = private_plane ? DRM_PLANE_TYPE_PRIMARY : DRM_PLANE_TYPE_OVERLAY;
+        drm_universal_plane_init(dev, plane, 0xff, &mdp5_plane_funcs,
+                                 mdp5_plane->formats, mdp5_plane->nformats,
+                                 type);
 
         mdp5_plane_install_properties(plane, &plane->base);
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp_kms.c b/drivers/gpu/drm/msm/mdp/mdp_kms.c
index 3be48f7c36be..03455b64a245 100644
--- a/drivers/gpu/drm/msm/mdp/mdp_kms.c
+++ b/drivers/gpu/drm/msm/mdp/mdp_kms.c
@@ -101,7 +101,8 @@ void mdp_irq_wait(struct mdp_kms *mdp_kms, uint32_t irqmask)
                 .count = 1,
         };
         mdp_irq_register(mdp_kms, &wait.irq);
-        wait_event(wait_event, (wait.count <= 0));
+        wait_event_timeout(wait_event, (wait.count <= 0),
+                        msecs_to_jiffies(100));
         mdp_irq_unregister(mdp_kms, &wait.irq);
 }
 
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index e6adafc7eff3..f9de156b9e65 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -56,6 +56,10 @@ static char *vram;
 MODULE_PARM_DESC(vram, "Configure VRAM size (for devices without IOMMU/GPUMMU");
 module_param(vram, charp, 0);
 
+/*
+ * Util/helpers:
+ */
+
 void __iomem *msm_ioremap(struct platform_device *pdev, const char *name,
                 const char *dbgname)
 {
@@ -143,6 +147,8 @@ static int msm_unload(struct drm_device *dev)
                                 priv->vram.paddr, &attrs);
         }
 
+        component_unbind_all(dev->dev, dev);
+
         dev->dev_private = NULL;
 
         kfree(priv);
@@ -175,6 +181,7 @@ static int msm_load(struct drm_device *dev, unsigned long flags)
         struct msm_kms *kms;
         int ret;
 
+
         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
         if (!priv) {
                 dev_err(dev->dev, "failed to allocate private data\n");
@@ -226,6 +233,13 @@ static int msm_load(struct drm_device *dev, unsigned long flags)
                                 (uint32_t)(priv->vram.paddr + size));
         }
 
+        platform_set_drvdata(pdev, dev);
+
+        /* Bind all our sub-components: */
+        ret = component_bind_all(dev->dev, dev);
+        if (ret)
+                return ret;
+
         switch (get_mdp_ver(pdev)) {
         case 4:
                 kms = mdp4_kms_init(dev);
@@ -281,8 +295,6 @@ static int msm_load(struct drm_device *dev, unsigned long flags)
                 goto fail;
         }
 
-        platform_set_drvdata(pdev, dev);
-
 #ifdef CONFIG_DRM_MSM_FBDEV
         priv->fbdev = msm_fbdev_init(dev);
 #endif
@@ -311,7 +323,6 @@ static void load_gpu(struct drm_device *dev)
                 gpu = NULL;
                 /* not fatal */
         }
-        mutex_unlock(&dev->struct_mutex);
 
         if (gpu) {
                 int ret;
@@ -321,10 +332,16 @@ static void load_gpu(struct drm_device *dev)
                         dev_err(dev->dev, "gpu hw init failed: %d\n", ret);
                         gpu->funcs->destroy(gpu);
                         gpu = NULL;
+                } else {
+                        /* give inactive pm a chance to kick in: */
+                        msm_gpu_retire(gpu);
                 }
+
         }
 
         priv->gpu = gpu;
+
+        mutex_unlock(&dev->struct_mutex);
 }
 
 static int msm_open(struct drm_device *dev, struct drm_file *file)
@@ -647,6 +664,12 @@ static int msm_ioctl_gem_new(struct drm_device *dev, void *data,
                 struct drm_file *file)
 {
         struct drm_msm_gem_new *args = data;
+
+        if (args->flags & ~MSM_BO_FLAGS) {
+                DRM_ERROR("invalid flags: %08x\n", args->flags);
+                return -EINVAL;
+        }
+
         return msm_gem_new_handle(dev, file, args->size,
                         args->flags, &args->handle);
 }
@@ -660,6 +683,11 @@ static int msm_ioctl_gem_cpu_prep(struct drm_device *dev, void *data,
         struct drm_gem_object *obj;
         int ret;
 
+        if (args->op & ~MSM_PREP_FLAGS) {
+                DRM_ERROR("invalid op: %08x\n", args->op);
+                return -EINVAL;
+        }
+
         obj = drm_gem_object_lookup(dev, file, args->handle);
         if (!obj)
                 return -ENOENT;
@@ -714,7 +742,14 @@ static int msm_ioctl_wait_fence(struct drm_device *dev, void *data,
                 struct drm_file *file)
 {
         struct drm_msm_wait_fence *args = data;
-        return msm_wait_fence_interruptable(dev, args->fence, &TS(args->timeout));
+
+        if (args->pad) {
+                DRM_ERROR("invalid pad: %08x\n", args->pad);
+                return -EINVAL;
+        }
+
+        return msm_wait_fence_interruptable(dev, args->fence,
+                        &TS(args->timeout));
 }
 
 static const struct drm_ioctl_desc msm_ioctls[] = {
@@ -819,18 +854,110 @@ static const struct dev_pm_ops msm_pm_ops = {
 };
 
 /*
+ * Componentized driver support:
+ */
+
+#ifdef CONFIG_OF
+/* NOTE: the CONFIG_OF case duplicates the same code as exynos or imx
+ * (or probably any other).. so probably some room for some helpers
+ */
+static int compare_of(struct device *dev, void *data)
+{
+        return dev->of_node == data;
+}
+
+static int msm_drm_add_components(struct device *master, struct master *m)
+{
+        struct device_node *np = master->of_node;
+        unsigned i;
+        int ret;
+
+        for (i = 0; ; i++) {
+                struct device_node *node;
+
+                node = of_parse_phandle(np, "connectors", i);
+                if (!node)
+                        break;
+
+                ret = component_master_add_child(m, compare_of, node);
+                of_node_put(node);
+
+                if (ret)
+                        return ret;
+        }
+        return 0;
+}
+#else
+static int compare_dev(struct device *dev, void *data)
+{
+        return dev == data;
+}
+
+static int msm_drm_add_components(struct device *master, struct master *m)
+{
+        /* For non-DT case, it kinda sucks.  We don't actually have a way
+         * to know whether or not we are waiting for certain devices (or if
+         * they are simply not present).  But for non-DT we only need to
+         * care about apq8064/apq8060/etc (all mdp4/a3xx):
+         */
+        static const char *devnames[] = {
+                        "hdmi_msm.0", "kgsl-3d0.0",
+        };
+        int i;
+
+        DBG("Adding components..");
+
+        for (i = 0; i < ARRAY_SIZE(devnames); i++) {
+                struct device *dev;
+                int ret;
+
+                dev = bus_find_device_by_name(&platform_bus_type,
+                                NULL, devnames[i]);
+                if (!dev) {
+                        dev_info(master, "still waiting for %s\n", devnames[i]);
+                        return -EPROBE_DEFER;
+                }
+
+                ret = component_master_add_child(m, compare_dev, dev);
+                if (ret) {
+                        DBG("could not add child: %d", ret);
+                        return ret;
+                }
+        }
+
+        return 0;
+}
+#endif
+
+static int msm_drm_bind(struct device *dev)
+{
+        return drm_platform_init(&msm_driver, to_platform_device(dev));
+}
+
+static void msm_drm_unbind(struct device *dev)
+{
+        drm_put_dev(platform_get_drvdata(to_platform_device(dev)));
+}
+
+static const struct component_master_ops msm_drm_ops = {
+                .add_components = msm_drm_add_components,
+                .bind = msm_drm_bind,
+                .unbind = msm_drm_unbind,
+};
+
+/*
  * Platform driver:
  */
 
 static int msm_pdev_probe(struct platform_device *pdev)
 {
         pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
-        return drm_platform_init(&msm_driver, pdev);
+        return component_master_add(&pdev->dev, &msm_drm_ops);
 }
 
 static int msm_pdev_remove(struct platform_device *pdev)
 {
-        drm_put_dev(platform_get_drvdata(pdev));
+        component_master_del(&pdev->dev, &msm_drm_ops);
 
         return 0;
 }
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index 3d63269c5b29..9d10ee0b5aac 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -22,6 +22,7 @@
 #include <linux/clk.h>
 #include <linux/cpufreq.h>
 #include <linux/module.h>
+#include <linux/component.h>
 #include <linux/platform_device.h>
 #include <linux/pm.h>
 #include <linux/pm_runtime.h>
@@ -69,6 +70,9 @@ struct msm_drm_private {
 
         struct msm_kms *kms;
 
+        /* subordinate devices, if present: */
+        struct platform_device *hdmi_pdev, *gpu_pdev;
+
         /* when we have more than one 'msm_gpu' these need to be an array: */
         struct msm_gpu *gpu;
         struct msm_file_private *lastctx;
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
index 5423e914e491..1f1f4cffdaed 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -23,7 +23,6 @@
  * Cmdstream submission:
  */
 
-#define BO_INVALID_FLAGS ~(MSM_SUBMIT_BO_READ | MSM_SUBMIT_BO_WRITE)
 /* make sure these don't conflict w/ MSM_SUBMIT_BO_x */
 #define BO_VALID    0x8000
 #define BO_LOCKED   0x4000
@@ -77,7 +76,7 @@ static int submit_lookup_objects(struct msm_gem_submit *submit,
                         goto out_unlock;
                 }
 
-                if (submit_bo.flags & BO_INVALID_FLAGS) {
+                if (submit_bo.flags & ~MSM_SUBMIT_BO_FLAGS) {
                         DRM_ERROR("invalid flags: %x\n", submit_bo.flags);
                         ret = -EINVAL;
                         goto out_unlock;
@@ -369,6 +368,18 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
                         goto out;
                 }
 
+                /* validate input from userspace: */
+                switch (submit_cmd.type) {
+                case MSM_SUBMIT_CMD_BUF:
+                case MSM_SUBMIT_CMD_IB_TARGET_BUF:
+                case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
+                        break;
+                default:
+                        DRM_ERROR("invalid type: %08x\n", submit_cmd.type);
+                        ret = -EINVAL;
+                        goto out;
+                }
+
                 ret = submit_bo(submit, submit_cmd.submit_idx,
                                 &msm_obj, &iova, NULL);
                 if (ret)
diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index 0cfe3f426ee4..3e667ca1f2b9 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -154,9 +154,18 @@ static int disable_axi(struct msm_gpu *gpu)
 
 int msm_gpu_pm_resume(struct msm_gpu *gpu)
 {
+        struct drm_device *dev = gpu->dev;
         int ret;
 
-        DBG("%s", gpu->name);
+        DBG("%s: active_cnt=%d", gpu->name, gpu->active_cnt);
+
+        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+
+        if (gpu->active_cnt++ > 0)
+                return 0;
+
+        if (WARN_ON(gpu->active_cnt <= 0))
+                return -EINVAL;
 
         ret = enable_pwrrail(gpu);
         if (ret)
@@ -175,9 +184,18 @@ int msm_gpu_pm_resume(struct msm_gpu *gpu)
 
 int msm_gpu_pm_suspend(struct msm_gpu *gpu)
 {
+        struct drm_device *dev = gpu->dev;
         int ret;
 
-        DBG("%s", gpu->name);
+        DBG("%s: active_cnt=%d", gpu->name, gpu->active_cnt);
+
+        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+
+        if (--gpu->active_cnt > 0)
+                return 0;
+
+        if (WARN_ON(gpu->active_cnt < 0))
+                return -EINVAL;
 
         ret = disable_axi(gpu);
         if (ret)
@@ -195,6 +213,55 @@ int msm_gpu_pm_suspend(struct msm_gpu *gpu)
 }
 
 /*
+ * Inactivity detection (for suspend):
+ */
+
+static void inactive_worker(struct work_struct *work)
+{
+        struct msm_gpu *gpu = container_of(work, struct msm_gpu, inactive_work);
+        struct drm_device *dev = gpu->dev;
+
+        if (gpu->inactive)
+                return;
+
+        DBG("%s: inactive!\n", gpu->name);
+        mutex_lock(&dev->struct_mutex);
+        if (!(msm_gpu_active(gpu) || gpu->inactive)) {
+                disable_axi(gpu);
+                disable_clk(gpu);
+                gpu->inactive = true;
+        }
+        mutex_unlock(&dev->struct_mutex);
+}
+
+static void inactive_handler(unsigned long data)
+{
+        struct msm_gpu *gpu = (struct msm_gpu *)data;
+        struct msm_drm_private *priv = gpu->dev->dev_private;
+
+        queue_work(priv->wq, &gpu->inactive_work);
+}
+
+/* cancel inactive timer and make sure we are awake: */
+static void inactive_cancel(struct msm_gpu *gpu)
+{
+        DBG("%s", gpu->name);
+        del_timer(&gpu->inactive_timer);
+        if (gpu->inactive) {
+                enable_clk(gpu);
+                enable_axi(gpu);
+                gpu->inactive = false;
+        }
+}
+
+static void inactive_start(struct msm_gpu *gpu)
+{
+        DBG("%s", gpu->name);
+        mod_timer(&gpu->inactive_timer,
+                        round_jiffies_up(jiffies + DRM_MSM_INACTIVE_JIFFIES));
+}
+
+/*
  * Hangcheck detection for locked gpu:
  */
 
@@ -206,7 +273,10 @@ static void recover_worker(struct work_struct *work)
         dev_err(dev->dev, "%s: hangcheck recover!\n", gpu->name);
 
         mutex_lock(&dev->struct_mutex);
-        gpu->funcs->recover(gpu);
+        if (msm_gpu_active(gpu)) {
+                inactive_cancel(gpu);
+                gpu->funcs->recover(gpu);
+        }
         mutex_unlock(&dev->struct_mutex);
 
         msm_gpu_retire(gpu);
@@ -281,6 +351,9 @@ static void retire_worker(struct work_struct *work)
         }
 
         mutex_unlock(&dev->struct_mutex);
+
+        if (!msm_gpu_active(gpu))
+                inactive_start(gpu);
 }
 
 /* call from irq handler to schedule work to retire bo's */
@@ -302,6 +375,8 @@ int msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
 
         gpu->submitted_fence = submit->fence;
 
+        inactive_cancel(gpu);
+
         ret = gpu->funcs->submit(gpu, submit, ctx);
         priv->lastctx = ctx;
 
@@ -357,11 +432,15 @@ int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev,
         gpu->dev = drm;
         gpu->funcs = funcs;
         gpu->name = name;
+        gpu->inactive = true;
 
         INIT_LIST_HEAD(&gpu->active_list);
         INIT_WORK(&gpu->retire_work, retire_worker);
+        INIT_WORK(&gpu->inactive_work, inactive_worker);
         INIT_WORK(&gpu->recover_work, recover_worker);
 
+        setup_timer(&gpu->inactive_timer, inactive_handler,
+                        (unsigned long)gpu);
         setup_timer(&gpu->hangcheck_timer, hangcheck_handler,
                         (unsigned long)gpu);
 
diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h
index 458db8c64c28..fad27008922f 100644
--- a/drivers/gpu/drm/msm/msm_gpu.h
+++ b/drivers/gpu/drm/msm/msm_gpu.h
@@ -72,6 +72,10 @@ struct msm_gpu {
 
         uint32_t submitted_fence;
 
+        /* is gpu powered/active? */
+        int active_cnt;
+        bool inactive;
+
         /* worker for handling active-list retiring: */
         struct work_struct retire_work;
 
@@ -91,7 +95,12 @@ struct msm_gpu {
         uint32_t bsc;
 #endif
 
-        /* Hang Detction: */
+        /* Hang and Inactivity Detection:
+         */
+#define DRM_MSM_INACTIVE_PERIOD   66 /* in ms (roughly four frames) */
+#define DRM_MSM_INACTIVE_JIFFIES  msecs_to_jiffies(DRM_MSM_INACTIVE_PERIOD)
+        struct timer_list inactive_timer;
+        struct work_struct inactive_work;
 #define DRM_MSM_HANGCHECK_PERIOD 500 /* in ms */
 #define DRM_MSM_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_MSM_HANGCHECK_PERIOD)
         struct timer_list hangcheck_timer;
@@ -99,6 +108,11 @@ struct msm_gpu {
         struct work_struct recover_work;
 };
 
+static inline bool msm_gpu_active(struct msm_gpu *gpu)
+{
+        return gpu->submitted_fence > gpu->funcs->last_fence(gpu);
+}
+
 static inline void gpu_write(struct msm_gpu *gpu, u32 reg, u32 data)
 {
         msm_writel(data, gpu->mmio + (reg << 2));
diff --git a/drivers/gpu/drm/nouveau/Makefile b/drivers/gpu/drm/nouveau/Makefile
index d310c195bdfe..b7d216264775 100644
--- a/drivers/gpu/drm/nouveau/Makefile
+++ b/drivers/gpu/drm/nouveau/Makefile
@@ -48,6 +48,7 @@ nouveau-y += core/subdev/bios/therm.o
 nouveau-y += core/subdev/bios/vmap.o
 nouveau-y += core/subdev/bios/volt.o
 nouveau-y += core/subdev/bios/xpio.o
+nouveau-y += core/subdev/bios/P0260.o
 nouveau-y += core/subdev/bus/hwsq.o
 nouveau-y += core/subdev/bus/nv04.o
 nouveau-y += core/subdev/bus/nv31.o
@@ -77,6 +78,7 @@ nouveau-y += core/subdev/devinit/nv98.o
 nouveau-y += core/subdev/devinit/nva3.o
 nouveau-y += core/subdev/devinit/nvaf.o
 nouveau-y += core/subdev/devinit/nvc0.o
+nouveau-y += core/subdev/devinit/gm107.o
 nouveau-y += core/subdev/fb/base.o
 nouveau-y += core/subdev/fb/nv04.o
 nouveau-y += core/subdev/fb/nv10.o
@@ -100,6 +102,7 @@ nouveau-y += core/subdev/fb/nvaa.o
 nouveau-y += core/subdev/fb/nvaf.o
 nouveau-y += core/subdev/fb/nvc0.o
 nouveau-y += core/subdev/fb/nve0.o
+nouveau-y += core/subdev/fb/gm107.o
 nouveau-y += core/subdev/fb/ramnv04.o
 nouveau-y += core/subdev/fb/ramnv10.o
 nouveau-y += core/subdev/fb/ramnv1a.o
@@ -114,6 +117,7 @@ nouveau-y += core/subdev/fb/ramnva3.o
 nouveau-y += core/subdev/fb/ramnvaa.o
 nouveau-y += core/subdev/fb/ramnvc0.o
 nouveau-y += core/subdev/fb/ramnve0.o
+nouveau-y += core/subdev/fb/ramgm107.o
 nouveau-y += core/subdev/fb/sddr3.o
 nouveau-y += core/subdev/fb/gddr5.o
 nouveau-y += core/subdev/gpio/base.o
@@ -136,7 +140,8 @@ nouveau-y += core/subdev/instmem/base.o
 nouveau-y += core/subdev/instmem/nv04.o
 nouveau-y += core/subdev/instmem/nv40.o
 nouveau-y += core/subdev/instmem/nv50.o
-nouveau-y += core/subdev/ltcg/nvc0.o
+nouveau-y += core/subdev/ltcg/gf100.o
+nouveau-y += core/subdev/ltcg/gm107.o
 nouveau-y += core/subdev/mc/base.o
 nouveau-y += core/subdev/mc/nv04.o
 nouveau-y += core/subdev/mc/nv40.o
@@ -170,6 +175,7 @@ nouveau-y += core/subdev/therm/nva3.o
 nouveau-y += core/subdev/therm/nvd0.o
 nouveau-y += core/subdev/timer/base.o
 nouveau-y += core/subdev/timer/nv04.o
+nouveau-y += core/subdev/timer/gk20a.o
 nouveau-y += core/subdev/vm/base.o
 nouveau-y += core/subdev/vm/nv04.o
 nouveau-y += core/subdev/vm/nv41.o
@@ -206,6 +212,7 @@ nouveau-y += core/engine/device/nv40.o
 nouveau-y += core/engine/device/nv50.o
 nouveau-y += core/engine/device/nvc0.o
 nouveau-y += core/engine/device/nve0.o
+nouveau-y += core/engine/device/gm100.o
 nouveau-y += core/engine/disp/base.o
 nouveau-y += core/engine/disp/nv04.o
 nouveau-y += core/engine/disp/nv50.o
@@ -216,6 +223,7 @@ nouveau-y += core/engine/disp/nva3.o
 nouveau-y += core/engine/disp/nvd0.o
 nouveau-y += core/engine/disp/nve0.o
 nouveau-y += core/engine/disp/nvf0.o
+nouveau-y += core/engine/disp/gm107.o
 nouveau-y += core/engine/disp/dacnv50.o
 nouveau-y += core/engine/disp/dport.o
 nouveau-y += core/engine/disp/hdanva3.o
@@ -242,13 +250,14 @@ nouveau-y += core/engine/graph/ctxnv40.o
 nouveau-y += core/engine/graph/ctxnv50.o
 nouveau-y += core/engine/graph/ctxnvc0.o
 nouveau-y += core/engine/graph/ctxnvc1.o
-nouveau-y += core/engine/graph/ctxnvc3.o
+nouveau-y += core/engine/graph/ctxnvc4.o
 nouveau-y += core/engine/graph/ctxnvc8.o
 nouveau-y += core/engine/graph/ctxnvd7.o
 nouveau-y += core/engine/graph/ctxnvd9.o
 nouveau-y += core/engine/graph/ctxnve4.o
 nouveau-y += core/engine/graph/ctxnvf0.o
 nouveau-y += core/engine/graph/ctxnv108.o
+nouveau-y += core/engine/graph/ctxgm107.o
 nouveau-y += core/engine/graph/nv04.o
 nouveau-y += core/engine/graph/nv10.o
 nouveau-y += core/engine/graph/nv20.o
@@ -261,13 +270,14 @@ nouveau-y += core/engine/graph/nv40.o
 nouveau-y += core/engine/graph/nv50.o
 nouveau-y += core/engine/graph/nvc0.o
 nouveau-y += core/engine/graph/nvc1.o
-nouveau-y += core/engine/graph/nvc3.o
+nouveau-y += core/engine/graph/nvc4.o
 nouveau-y += core/engine/graph/nvc8.o
 nouveau-y += core/engine/graph/nvd7.o
 nouveau-y += core/engine/graph/nvd9.o
 nouveau-y += core/engine/graph/nve4.o
 nouveau-y += core/engine/graph/nvf0.o
 nouveau-y += core/engine/graph/nv108.o
+nouveau-y += core/engine/graph/gm107.o
 nouveau-y += core/engine/mpeg/nv31.o
 nouveau-y += core/engine/mpeg/nv40.o
 nouveau-y += core/engine/mpeg/nv44.o
diff --git a/drivers/gpu/drm/nouveau/core/core/namedb.c b/drivers/gpu/drm/nouveau/core/core/namedb.c
index 1ce95a8709df..0594a599f6fb 100644
--- a/drivers/gpu/drm/nouveau/core/core/namedb.c
+++ b/drivers/gpu/drm/nouveau/core/core/namedb.c
@@ -167,7 +167,7 @@ int
 nouveau_namedb_create_(struct nouveau_object *parent,
                        struct nouveau_object *engine,
                        struct nouveau_oclass *oclass, u32 pclass,
-                       struct nouveau_oclass *sclass, u32 engcls,
+                       struct nouveau_oclass *sclass, u64 engcls,
                        int length, void **pobject)
 {
         struct nouveau_namedb *namedb;
diff --git a/drivers/gpu/drm/nouveau/core/core/parent.c b/drivers/gpu/drm/nouveau/core/core/parent.c
index 313380ce632d..dee5d1235e9b 100644
--- a/drivers/gpu/drm/nouveau/core/core/parent.c
+++ b/drivers/gpu/drm/nouveau/core/core/parent.c
@@ -49,7 +49,7 @@ nouveau_parent_sclass(struct nouveau_object *parent, u16 handle,
 
         mask = nv_parent(parent)->engine;
         while (mask) {
-                int i = ffsll(mask) - 1;
+                int i = __ffs64(mask);
 
                 if (nv_iclass(parent, NV_CLIENT_CLASS))
                         engine = nv_engine(nv_client(parent)->device);
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/base.c b/drivers/gpu/drm/nouveau/core/engine/device/base.c
index dd01c6c435d6..18c8c7245b73 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/base.c
@@ -131,8 +131,8 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
-        mmio_base = pci_resource_start(device->pdev, 0);
-        mmio_size = pci_resource_len(device->pdev, 0);
+        mmio_base = nv_device_resource_start(device, 0);
+        mmio_size = nv_device_resource_len(device, 0);
 
         /* translate api disable mask into internal mapping */
         disable = args->debug0;
@@ -185,6 +185,7 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
                         case 0x0e0:
                         case 0x0f0:
                         case 0x100: device->card_type = NV_E0; break;
+                        case 0x110: device->card_type = GM100; break;
                         default:
                                 break;
                         }
@@ -208,6 +209,7 @@ nouveau_devobj_ctor(struct nouveau_object *parent,
                 case NV_C0:
                 case NV_D0: ret = nvc0_identify(device); break;
                 case NV_E0: ret = nve0_identify(device); break;
+                case GM100: ret = gm100_identify(device); break;
                 default:
                         ret = -EINVAL;
                         break;
@@ -446,6 +448,72 @@ nouveau_device_dtor(struct nouveau_object *object)
         nouveau_engine_destroy(&device->base);
 }
 
+resource_size_t
+nv_device_resource_start(struct nouveau_device *device, unsigned int bar)
+{
+        if (nv_device_is_pci(device)) {
+                return pci_resource_start(device->pdev, bar);
+        } else {
+                struct resource *res;
+                res = platform_get_resource(device->platformdev,
+                                            IORESOURCE_MEM, bar);
+                if (!res)
+                        return 0;
+                return res->start;
+        }
+}
+
+resource_size_t
+nv_device_resource_len(struct nouveau_device *device, unsigned int bar)
+{
+        if (nv_device_is_pci(device)) {
+                return pci_resource_len(device->pdev, bar);
+        } else {
+                struct resource *res;
+                res = platform_get_resource(device->platformdev,
+                                            IORESOURCE_MEM, bar);
+                if (!res)
+                        return 0;
+                return resource_size(res);
+        }
+}
+
+dma_addr_t
+nv_device_map_page(struct nouveau_device *device, struct page *page)
+{
+        dma_addr_t ret;
+
+        if (nv_device_is_pci(device)) {
+                ret = pci_map_page(device->pdev, page, 0, PAGE_SIZE,
+                                   PCI_DMA_BIDIRECTIONAL);
+                if (pci_dma_mapping_error(device->pdev, ret))
+                        ret = 0;
+        } else {
+                ret = page_to_phys(page);
+        }
+
+        return ret;
+}
+
+void
+nv_device_unmap_page(struct nouveau_device *device, dma_addr_t addr)
+{
+        if (nv_device_is_pci(device))
+                pci_unmap_page(device->pdev, addr, PAGE_SIZE,
+                               PCI_DMA_BIDIRECTIONAL);
+}
+
+int
+nv_device_get_irq(struct nouveau_device *device, bool stall)
+{
+        if (nv_device_is_pci(device)) {
+                return device->pdev->irq;
+        } else {
+                return platform_get_irq_byname(device->platformdev,
+                                               stall ? "stall" : "nonstall");
+        }
+}
+
 static struct nouveau_oclass
 nouveau_device_oclass = {
         .handle = NV_ENGINE(DEVICE, 0x00),
@@ -457,8 +525,8 @@ nouveau_device_oclass = {
 };
 
 int
-nouveau_device_create_(struct pci_dev *pdev, u64 name, const char *sname,
-                       const char *cfg, const char *dbg,
+nouveau_device_create_(void *dev, enum nv_bus_type type, u64 name,
+                       const char *sname, const char *cfg, const char *dbg,
                        int length, void **pobject)
 {
         struct nouveau_device *device;
@@ -476,7 +544,14 @@ nouveau_device_create_(struct pci_dev *pdev, u64 name, const char *sname,
         if (ret)
                 goto done;
 
-        device->pdev = pdev;
+        switch (type) {
+        case NOUVEAU_BUS_PCI:
+                device->pdev = dev;
+                break;
+        case NOUVEAU_BUS_PLATFORM:
+                device->platformdev = dev;
+                break;
+        }
         device->handle = name;
         device->cfgopt = cfg;
         device->dbgopt = dbg;
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/gm100.c b/drivers/gpu/drm/nouveau/core/engine/device/gm100.c
new file mode 100644
index 000000000000..d258c21c4a22
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/device/gm100.c
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bus.h>
+#include <subdev/gpio.h>
+#include <subdev/i2c.h>
+#include <subdev/clock.h>
+#include <subdev/therm.h>
+#include <subdev/mxm.h>
+#include <subdev/devinit.h>
+#include <subdev/mc.h>
+#include <subdev/timer.h>
+#include <subdev/fb.h>
+#include <subdev/ltcg.h>
+#include <subdev/ibus.h>
+#include <subdev/instmem.h>
+#include <subdev/vm.h>
+#include <subdev/bar.h>
+#include <subdev/pwr.h>
+#include <subdev/volt.h>
+
+#include <engine/device.h>
+#include <engine/dmaobj.h>
+#include <engine/fifo.h>
+#include <engine/software.h>
+#include <engine/graph.h>
+#include <engine/disp.h>
+#include <engine/copy.h>
+#include <engine/bsp.h>
+#include <engine/vp.h>
+#include <engine/ppp.h>
+#include <engine/perfmon.h>
+
+int
+gm100_identify(struct nouveau_device *device)
+{
+        switch (device->chipset) {
+        case 0x117:
+                device->cname = "GM107";
+                device->oclass[NVDEV_SUBDEV_VBIOS  ] = &nouveau_bios_oclass;
+                device->oclass[NVDEV_SUBDEV_GPIO   ] = &nve0_gpio_oclass;
+                device->oclass[NVDEV_SUBDEV_I2C    ] = &nvd0_i2c_oclass;
+                device->oclass[NVDEV_SUBDEV_CLOCK  ] = &nve0_clock_oclass;
+#if 0
+                device->oclass[NVDEV_SUBDEV_THERM  ] = &nvd0_therm_oclass;
+#endif
+                device->oclass[NVDEV_SUBDEV_MXM    ] = &nv50_mxm_oclass;
+                device->oclass[NVDEV_SUBDEV_DEVINIT] =  gm107_devinit_oclass;
+                device->oclass[NVDEV_SUBDEV_MC     ] =  nvc3_mc_oclass;
+                device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
+                device->oclass[NVDEV_SUBDEV_TIMER  ] = &gk20a_timer_oclass;
+                device->oclass[NVDEV_SUBDEV_FB     ] =  gm107_fb_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gm107_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
+                device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
+                device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
+                device->oclass[NVDEV_SUBDEV_BAR    ] = &nvc0_bar_oclass;
+#if 0
+                device->oclass[NVDEV_SUBDEV_PWR    ] = &nv108_pwr_oclass;
+                device->oclass[NVDEV_SUBDEV_VOLT   ] = &nv40_volt_oclass;
+#endif
+                device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvd0_dmaeng_oclass;
+                device->oclass[NVDEV_ENGINE_FIFO   ] =  nv108_fifo_oclass;
+                device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
+                device->oclass[NVDEV_ENGINE_GR     ] =  gm107_graph_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  gm107_disp_oclass;
+                device->oclass[NVDEV_ENGINE_COPY0  ] = &nve0_copy0_oclass;
+#if 0
+                device->oclass[NVDEV_ENGINE_COPY1  ] = &nve0_copy1_oclass;
+#endif
+                device->oclass[NVDEV_ENGINE_COPY2  ] = &nve0_copy2_oclass;
+#if 0
+                device->oclass[NVDEV_ENGINE_BSP    ] = &nve0_bsp_oclass;
+                device->oclass[NVDEV_ENGINE_VP     ] = &nve0_vp_oclass;
+                device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
+#endif
+                break;
+        default:
+                nv_fatal(device, "unknown Maxwell chipset\n");
+                return -EINVAL;
+        }
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv04.c b/drivers/gpu/drm/nouveau/core/engine/device/nv04.c
index 32113b08c4d5..0a51ff4e9e00 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv04.c
@@ -60,7 +60,7 @@ nv04_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv04_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv04_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv04_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x05:
                 device->cname = "NV05";
@@ -78,7 +78,7 @@ nv04_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv04_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv04_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv04_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         default:
                 nv_fatal(device, "unknown RIVA chipset\n");
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv10.c b/drivers/gpu/drm/nouveau/core/engine/device/nv10.c
index 744f15d7e131..e008de8b51b0 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv10.c
@@ -60,7 +60,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nv04_vmmgr_oclass;
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nv04_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x15:
                 device->cname = "NV15";
@@ -79,7 +79,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv10_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x16:
                 device->cname = "NV16";
@@ -98,7 +98,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv10_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x1a:
                 device->cname = "nForce";
@@ -117,7 +117,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv10_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x11:
                 device->cname = "NV11";
@@ -136,7 +136,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv10_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x17:
                 device->cname = "NV17";
@@ -155,7 +155,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x1f:
                 device->cname = "nForce2";
@@ -174,7 +174,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x18:
                 device->cname = "NV18";
@@ -193,7 +193,7 @@ nv10_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv10_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         default:
                 nv_fatal(device, "unknown Celsius chipset\n");
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv20.c b/drivers/gpu/drm/nouveau/core/engine/device/nv20.c
index 27ba61fb2710..7b629a3aed05 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv20.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv20.c
@@ -63,7 +63,7 @@ nv20_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv20_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x25:
                 device->cname = "NV25";
@@ -82,7 +82,7 @@ nv20_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv25_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x28:
                 device->cname = "NV28";
@@ -101,7 +101,7 @@ nv20_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv25_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x2a:
                 device->cname = "NV2A";
@@ -120,7 +120,7 @@ nv20_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv2a_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         default:
                 nv_fatal(device, "unknown Kelvin chipset\n");
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv30.c b/drivers/gpu/drm/nouveau/core/engine/device/nv30.c
index fd47ace67543..7dfddd5a1908 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv30.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv30.c
@@ -63,7 +63,7 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv30_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x35:
                 device->cname = "NV35";
@@ -82,7 +82,7 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv17_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv35_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x31:
                 device->cname = "NV31";
@@ -102,7 +102,7 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv30_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv31_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x36:
                 device->cname = "NV36";
@@ -122,7 +122,7 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv35_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv31_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         case 0x34:
                 device->cname = "NV34";
@@ -142,7 +142,7 @@ nv30_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv34_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv31_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 break;
         default:
                 nv_fatal(device, "unknown Rankine chipset\n");
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv40.c b/drivers/gpu/drm/nouveau/core/engine/device/nv40.c
index 08b88591ed60..7c1ce6cf4f1f 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv40.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv40.c
@@ -70,7 +70,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x41:
@@ -93,7 +93,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x42:
@@ -116,7 +116,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x43:
@@ -139,7 +139,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv40_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x45:
@@ -162,7 +162,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x47:
@@ -185,7 +185,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x49:
@@ -208,7 +208,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x4b:
@@ -231,7 +231,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x44:
@@ -254,7 +254,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x46:
@@ -277,7 +277,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x4a:
@@ -300,7 +300,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x4c:
@@ -323,7 +323,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x4e:
@@ -346,7 +346,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x63:
@@ -369,7 +369,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x67:
@@ -392,7 +392,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         case 0x68:
@@ -415,7 +415,7 @@ nv40_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv10_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv40_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv44_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv04_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv04_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv40_perfmon_oclass;
                 break;
         default:
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nv50.c b/drivers/gpu/drm/nouveau/core/engine/device/nv50.c
index 81d5c26643d5..66499fa0f758 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nv50.c
@@ -79,7 +79,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_SW     ] =  nv50_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] = &nv50_graph_oclass;
                 device->oclass[NVDEV_ENGINE_MPEG   ] = &nv50_mpeg_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv50_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv50_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv50_perfmon_oclass;
                 break;
         case 0x84:
@@ -107,7 +107,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv84_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv84_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0x86:
@@ -135,7 +135,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv84_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv84_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0x92:
@@ -163,7 +163,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv84_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv84_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0x94:
@@ -191,7 +191,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv94_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv94_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0x96:
@@ -219,7 +219,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv94_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv94_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0x98:
@@ -247,7 +247,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv98_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv98_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv94_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv94_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0xa0:
@@ -275,7 +275,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_VP     ] = &nv84_vp_oclass;
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv84_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv84_bsp_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nva0_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nva0_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0xaa:
@@ -303,7 +303,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv98_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv98_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv94_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv94_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0xac:
@@ -331,7 +331,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_CRYPT  ] = &nv98_crypt_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv98_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nv94_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nv94_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nv84_perfmon_oclass;
                 break;
         case 0xa3:
@@ -361,7 +361,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv98_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nva3_copy_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nva3_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nva3_perfmon_oclass;
                 break;
         case 0xa5:
@@ -390,7 +390,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv98_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nva3_copy_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nva3_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nva3_perfmon_oclass;
                 break;
         case 0xa8:
@@ -419,7 +419,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv98_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nva3_copy_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nva3_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nva3_perfmon_oclass;
                 break;
         case 0xaf:
@@ -448,7 +448,7 @@ nv50_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nv98_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nv98_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nva3_copy_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nva3_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] =  nva3_perfmon_oclass;
                 break;
         default:
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/device/nvc0.c
index b7d66b59f43d..2075b3027052 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nvc0.c
@@ -70,7 +70,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
@@ -86,7 +86,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nva3_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xc4:
@@ -102,7 +102,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
@@ -112,13 +112,13 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
-                device->oclass[NVDEV_ENGINE_GR     ] =  nvc3_graph_oclass;
+                device->oclass[NVDEV_ENGINE_GR     ] =  nvc4_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nvc0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nva3_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xc3:
@@ -134,7 +134,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
@@ -144,12 +144,12 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
-                device->oclass[NVDEV_ENGINE_GR     ] =  nvc3_graph_oclass;
+                device->oclass[NVDEV_ENGINE_GR     ] =  nvc4_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nvc0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nva3_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xce:
@@ -165,7 +165,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
@@ -175,13 +175,13 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
-                device->oclass[NVDEV_ENGINE_GR     ] =  nvc3_graph_oclass;
+                device->oclass[NVDEV_ENGINE_GR     ] =  nvc4_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nvc0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nva3_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xcf:
@@ -197,7 +197,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
@@ -207,13 +207,13 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_DMAOBJ ] = &nvc0_dmaeng_oclass;
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nvc0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
-                device->oclass[NVDEV_ENGINE_GR     ] =  nvc3_graph_oclass;
+                device->oclass[NVDEV_ENGINE_GR     ] =  nvc4_graph_oclass;
                 device->oclass[NVDEV_ENGINE_VP     ] = &nvc0_vp_oclass;
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nva3_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xc1:
@@ -229,7 +229,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
@@ -244,7 +244,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nva3_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xc8:
@@ -260,7 +260,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
@@ -276,7 +276,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nvc0_copy1_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nva3_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nva3_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xd9:
@@ -292,7 +292,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
@@ -307,7 +307,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nvd0_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nvd0_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         case 0xd7:
@@ -323,7 +323,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nvc0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nvc0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
@@ -336,7 +336,7 @@ nvc0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_BSP    ] = &nvc0_bsp_oclass;
                 device->oclass[NVDEV_ENGINE_PPP    ] = &nvc0_ppp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nvc0_copy0_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nvd0_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nvd0_disp_oclass;
                 device->oclass[NVDEV_ENGINE_PERFMON] = &nvc0_perfmon_oclass;
                 break;
         default:
diff --git a/drivers/gpu/drm/nouveau/core/engine/device/nve0.c b/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
index 987edbc30a09..9784cbf8a9d2 100644
--- a/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/device/nve0.c
@@ -70,7 +70,7 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
@@ -81,7 +81,7 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nve0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nve4_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nve0_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nve0_disp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nve0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nve0_copy1_oclass;
                 device->oclass[NVDEV_ENGINE_COPY2  ] = &nve0_copy2_oclass;
@@ -103,7 +103,7 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
@@ -114,7 +114,7 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nve0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nve4_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nve0_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nve0_disp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nve0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nve0_copy1_oclass;
                 device->oclass[NVDEV_ENGINE_COPY2  ] = &nve0_copy2_oclass;
@@ -136,7 +136,7 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
@@ -147,7 +147,7 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nve0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nve4_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nve0_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nve0_disp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nve0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nve0_copy1_oclass;
                 device->oclass[NVDEV_ENGINE_COPY2  ] = &nve0_copy2_oclass;
@@ -169,7 +169,7 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
@@ -180,7 +180,7 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nve0_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nvf0_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nvf0_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nvf0_disp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nve0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nve0_copy1_oclass;
                 device->oclass[NVDEV_ENGINE_COPY2  ] = &nve0_copy2_oclass;
@@ -204,7 +204,7 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_SUBDEV_BUS    ] =  nvc0_bus_oclass;
                 device->oclass[NVDEV_SUBDEV_TIMER  ] = &nv04_timer_oclass;
                 device->oclass[NVDEV_SUBDEV_FB     ] =  nve0_fb_oclass;
-                device->oclass[NVDEV_SUBDEV_LTCG   ] = &nvc0_ltcg_oclass;
+                device->oclass[NVDEV_SUBDEV_LTCG   ] =  gf100_ltcg_oclass;
                 device->oclass[NVDEV_SUBDEV_IBUS   ] = &nve0_ibus_oclass;
                 device->oclass[NVDEV_SUBDEV_INSTMEM] =  nv50_instmem_oclass;
                 device->oclass[NVDEV_SUBDEV_VM     ] = &nvc0_vmmgr_oclass;
@@ -215,7 +215,7 @@ nve0_identify(struct nouveau_device *device)
                 device->oclass[NVDEV_ENGINE_FIFO   ] =  nv108_fifo_oclass;
                 device->oclass[NVDEV_ENGINE_SW     ] =  nvc0_software_oclass;
                 device->oclass[NVDEV_ENGINE_GR     ] =  nv108_graph_oclass;
-                device->oclass[NVDEV_ENGINE_DISP   ] = &nvf0_disp_oclass;
+                device->oclass[NVDEV_ENGINE_DISP   ] =  nvf0_disp_oclass;
                 device->oclass[NVDEV_ENGINE_COPY0  ] = &nve0_copy0_oclass;
                 device->oclass[NVDEV_ENGINE_COPY1  ] = &nve0_copy1_oclass;
                 device->oclass[NVDEV_ENGINE_COPY2  ] = &nve0_copy2_oclass;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/dport.c b/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
index 1bd4c63369c1..3ca2d25b7f5e 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/dport.c
@@ -273,7 +273,7 @@ nouveau_dp_train(struct nouveau_disp *disp, const struct nouveau_dp_func *func,
                 .outp = outp,
                 .head = head,
         }, *dp = &_dp;
-        const u32 bw_list[] = { 270000, 162000, 0 };
+        const u32 bw_list[] = { 540000, 270000, 162000, 0 };
         const u32 *link_bw = bw_list;
         u8  hdr, cnt, len;
         u32 data;
@@ -312,6 +312,14 @@ nouveau_dp_train(struct nouveau_disp *disp, const struct nouveau_dp_func *func,
                 ERR("failed to read DPCD\n");
         }
 
+        /* bring capabilities within encoder limits */
+        if ((dp->dpcd[2] & 0x1f) > dp->outp->dpconf.link_nr) {
+                dp->dpcd[2] &= ~0x1f;
+                dp->dpcd[2] |= dp->outp->dpconf.link_nr;
+        }
+        if (dp->dpcd[1] > dp->outp->dpconf.link_bw)
+                dp->dpcd[1] = dp->outp->dpconf.link_bw;
+
         /* adjust required bandwidth for 8B/10B coding overhead */
         datarate = (datarate / 8) * 10;
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c b/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c
new file mode 100644
index 000000000000..cf6f59677b74
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/gm107.c
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <engine/software.h>
+#include <engine/disp.h>
+
+#include <core/class.h>
+
+#include "nv50.h"
+
+/*******************************************************************************
+ * Base display object
+ ******************************************************************************/
+
+static struct nouveau_oclass
+gm107_disp_sclass[] = {
+        { GM107_DISP_MAST_CLASS, &nvd0_disp_mast_ofuncs },
+        { GM107_DISP_SYNC_CLASS, &nvd0_disp_sync_ofuncs },
+        { GM107_DISP_OVLY_CLASS, &nvd0_disp_ovly_ofuncs },
+        { GM107_DISP_OIMM_CLASS, &nvd0_disp_oimm_ofuncs },
+        { GM107_DISP_CURS_CLASS, &nvd0_disp_curs_ofuncs },
+        {}
+};
+
+static struct nouveau_oclass
+gm107_disp_base_oclass[] = {
+        { GM107_DISP_CLASS, &nvd0_disp_base_ofuncs, nvd0_disp_base_omthds },
+        {}
+};
+
+/*******************************************************************************
+ * Display engine implementation
+ ******************************************************************************/
+
+static int
+gm107_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+               struct nouveau_oclass *oclass, void *data, u32 size,
+               struct nouveau_object **pobject)
+{
+        struct nv50_disp_priv *priv;
+        int heads = nv_rd32(parent, 0x022448);
+        int ret;
+
+        ret = nouveau_disp_create(parent, engine, oclass, heads,
+                                  "PDISP", "display", &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        nv_engine(priv)->sclass = gm107_disp_base_oclass;
+        nv_engine(priv)->cclass = &nv50_disp_cclass;
+        nv_subdev(priv)->intr = nvd0_disp_intr;
+        INIT_WORK(&priv->supervisor, nvd0_disp_intr_supervisor);
+        priv->sclass = gm107_disp_sclass;
+        priv->head.nr = heads;
+        priv->dac.nr = 3;
+        priv->sor.nr = 4;
+        priv->dac.power = nv50_dac_power;
+        priv->dac.sense = nv50_dac_sense;
+        priv->sor.power = nv50_sor_power;
+        priv->sor.hda_eld = nvd0_hda_eld;
+        priv->sor.hdmi = nvd0_hdmi_ctrl;
+        priv->sor.dp = &nvd0_sor_dp_func;
+        return 0;
+}
+
+struct nouveau_oclass *
+gm107_disp_oclass = &(struct nv50_disp_impl) {
+        .base.base.handle = NV_ENGINE(DISP, 0x07),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = gm107_disp_ctor,
+                .dtor = _nouveau_disp_dtor,
+                .init = _nouveau_disp_init,
+                .fini = _nouveau_disp_fini,
+        },
+        .mthd.core = &nve0_disp_mast_mthd_chan,
+        .mthd.base = &nvd0_disp_sync_mthd_chan,
+        .mthd.ovly = &nve0_disp_ovly_mthd_chan,
+        .mthd.prev = -0x020000,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c
index 7cf8b1348632..6c89af792889 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv04.c
@@ -22,7 +22,7 @@
  * Authors: Ben Skeggs
  */
 
-#include <engine/disp.h>
+#include "priv.h"
 
 #include <core/event.h>
 #include <core/class.h>
@@ -138,13 +138,13 @@ nv04_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-struct nouveau_oclass
-nv04_disp_oclass = {
-        .handle = NV_ENGINE(DISP, 0x04),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nv04_disp_oclass = &(struct nouveau_disp_impl) {
+        .base.handle = NV_ENGINE(DISP, 0x04),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv04_disp_ctor,
                 .dtor = _nouveau_disp_dtor,
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
-};
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c
index 9ad722e4e087..9a0cab9c3adb 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.c
@@ -26,8 +26,7 @@
 #include <core/parent.h>
 #include <core/handle.h>
 #include <core/class.h>
-
-#include <engine/disp.h>
+#include <core/enum.h>
 
 #include <subdev/bios.h>
 #include <subdev/bios/dcb.h>
@@ -227,6 +226,177 @@ nv50_disp_dmac_fini(struct nouveau_object *object, bool suspend)
  * EVO master channel object
  ******************************************************************************/
 
+static void
+nv50_disp_mthd_list(struct nv50_disp_priv *priv, int debug, u32 base, int c,
+                    const struct nv50_disp_mthd_list *list, int inst)
+{
+        struct nouveau_object *disp = nv_object(priv);
+        int i;
+
+        for (i = 0; list->data[i].mthd; i++) {
+                if (list->data[i].addr) {
+                        u32 next = nv_rd32(priv, list->data[i].addr + base + 0);
+                        u32 prev = nv_rd32(priv, list->data[i].addr + base + c);
+                        u32 mthd = list->data[i].mthd + (list->mthd * inst);
+                        const char *name = list->data[i].name;
+                        char mods[16];
+
+                        if (prev != next)
+                                snprintf(mods, sizeof(mods), "-> 0x%08x", next);
+                        else
+                                snprintf(mods, sizeof(mods), "%13c", ' ');
+
+                        nv_printk_(disp, debug, "\t0x%04x: 0x%08x %s%s%s\n",
+                                   mthd, prev, mods, name ? " // " : "",
+                                   name ? name : "");
+                }
+        }
+}
+
+void
+nv50_disp_mthd_chan(struct nv50_disp_priv *priv, int debug, int head,
+                    const struct nv50_disp_mthd_chan *chan)
+{
+        struct nouveau_object *disp = nv_object(priv);
+        const struct nv50_disp_impl *impl = (void *)disp->oclass;
+        const struct nv50_disp_mthd_list *list;
+        int i, j;
+
+        if (debug > nv_subdev(priv)->debug)
+                return;
+
+        for (i = 0; (list = chan->data[i].mthd) != NULL; i++) {
+                u32 base = head * chan->addr;
+                for (j = 0; j < chan->data[i].nr; j++, base += list->addr) {
+                        const char *cname = chan->name;
+                        const char *sname = "";
+                        char cname_[16], sname_[16];
+
+                        if (chan->addr) {
+                                snprintf(cname_, sizeof(cname_), "%s %d",
+                                         chan->name, head);
+                                cname = cname_;
+                        }
+
+                        if (chan->data[i].nr > 1) {
+                                snprintf(sname_, sizeof(sname_), " - %s %d",
+                                         chan->data[i].name, j);
+                                sname = sname_;
+                        }
+
+                        nv_printk_(disp, debug, "%s%s:\n", cname, sname);
+                        nv50_disp_mthd_list(priv, debug, base, impl->mthd.prev,
+                                            list, j);
+                }
+        }
+}
+
+const struct nv50_disp_mthd_list
+nv50_disp_mast_mthd_base = {
+        .mthd = 0x0000,
+        .addr = 0x000000,
+        .data = {
+                { 0x0080, 0x000000 },
+                { 0x0084, 0x610bb8 },
+                { 0x0088, 0x610b9c },
+                { 0x008c, 0x000000 },
+                {}
+        }
+};
+
+static const struct nv50_disp_mthd_list
+nv50_disp_mast_mthd_dac = {
+        .mthd = 0x0080,
+        .addr = 0x000008,
+        .data = {
+                { 0x0400, 0x610b58 },
+                { 0x0404, 0x610bdc },
+                { 0x0420, 0x610828 },
+                {}
+        }
+};
+
+const struct nv50_disp_mthd_list
+nv50_disp_mast_mthd_sor = {
+        .mthd = 0x0040,
+        .addr = 0x000008,
+        .data = {
+                { 0x0600, 0x610b70 },
+                {}
+        }
+};
+
+const struct nv50_disp_mthd_list
+nv50_disp_mast_mthd_pior = {
+        .mthd = 0x0040,
+        .addr = 0x000008,
+        .data = {
+                { 0x0700, 0x610b80 },
+                {}
+        }
+};
+
+static const struct nv50_disp_mthd_list
+nv50_disp_mast_mthd_head = {
+        .mthd = 0x0400,
+        .addr = 0x000540,
+        .data = {
+                { 0x0800, 0x610ad8 },
+                { 0x0804, 0x610ad0 },
+                { 0x0808, 0x610a48 },
+                { 0x080c, 0x610a78 },
+                { 0x0810, 0x610ac0 },
+                { 0x0814, 0x610af8 },
+                { 0x0818, 0x610b00 },
+                { 0x081c, 0x610ae8 },
+                { 0x0820, 0x610af0 },
+                { 0x0824, 0x610b08 },
+                { 0x0828, 0x610b10 },
+                { 0x082c, 0x610a68 },
+                { 0x0830, 0x610a60 },
+                { 0x0834, 0x000000 },
+                { 0x0838, 0x610a40 },
+                { 0x0840, 0x610a24 },
+                { 0x0844, 0x610a2c },
+                { 0x0848, 0x610aa8 },
+                { 0x084c, 0x610ab0 },
+                { 0x0860, 0x610a84 },
+                { 0x0864, 0x610a90 },
+                { 0x0868, 0x610b18 },
+                { 0x086c, 0x610b20 },
+                { 0x0870, 0x610ac8 },
+                { 0x0874, 0x610a38 },
+                { 0x0880, 0x610a58 },
+                { 0x0884, 0x610a9c },
+                { 0x08a0, 0x610a70 },
+                { 0x08a4, 0x610a50 },
+                { 0x08a8, 0x610ae0 },
+                { 0x08c0, 0x610b28 },
+                { 0x08c4, 0x610b30 },
+                { 0x08c8, 0x610b40 },
+                { 0x08d4, 0x610b38 },
+                { 0x08d8, 0x610b48 },
+                { 0x08dc, 0x610b50 },
+                { 0x0900, 0x610a18 },
+                { 0x0904, 0x610ab8 },
+                {}
+        }
+};
+
+static const struct nv50_disp_mthd_chan
+nv50_disp_mast_mthd_chan = {
+        .name = "Core",
+        .addr = 0x000000,
+        .data = {
+                { "Global", 1, &nv50_disp_mast_mthd_base },
+                {    "DAC", 3, &nv50_disp_mast_mthd_dac  },
+                {    "SOR", 2, &nv50_disp_mast_mthd_sor  },
+                {   "PIOR", 3, &nv50_disp_mast_mthd_pior },
+                {   "HEAD", 2, &nv50_disp_mast_mthd_head },
+                {}
+        }
+};
+
 static int
 nv50_disp_mast_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
@@ -323,6 +493,56 @@ nv50_disp_mast_ofuncs = {
  * EVO sync channel objects
  ******************************************************************************/
 
+static const struct nv50_disp_mthd_list
+nv50_disp_sync_mthd_base = {
+        .mthd = 0x0000,
+        .addr = 0x000000,
+        .data = {
+                { 0x0080, 0x000000 },
+                { 0x0084, 0x0008c4 },
+                { 0x0088, 0x0008d0 },
+                { 0x008c, 0x0008dc },
+                { 0x0090, 0x0008e4 },
+                { 0x0094, 0x610884 },
+                { 0x00a0, 0x6108a0 },
+                { 0x00a4, 0x610878 },
+                { 0x00c0, 0x61086c },
+                { 0x00e0, 0x610858 },
+                { 0x00e4, 0x610860 },
+                { 0x00e8, 0x6108ac },
+                { 0x00ec, 0x6108b4 },
+                { 0x0100, 0x610894 },
+                { 0x0110, 0x6108bc },
+                { 0x0114, 0x61088c },
+                {}
+        }
+};
+
+const struct nv50_disp_mthd_list
+nv50_disp_sync_mthd_image = {
+        .mthd = 0x0400,
+        .addr = 0x000000,
+        .data = {
+                { 0x0800, 0x6108f0 },
+                { 0x0804, 0x6108fc },
+                { 0x0808, 0x61090c },
+                { 0x080c, 0x610914 },
+                { 0x0810, 0x610904 },
+                {}
+        }
+};
+
+static const struct nv50_disp_mthd_chan
+nv50_disp_sync_mthd_chan = {
+        .name = "Base",
+        .addr = 0x000540,
+        .data = {
+                { "Global", 1, &nv50_disp_sync_mthd_base },
+                {  "Image", 2, &nv50_disp_sync_mthd_image },
+                {}
+        }
+};
+
 static int
 nv50_disp_sync_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
@@ -362,6 +582,44 @@ nv50_disp_sync_ofuncs = {
  * EVO overlay channel objects
  ******************************************************************************/
 
+const struct nv50_disp_mthd_list
+nv50_disp_ovly_mthd_base = {
+        .mthd = 0x0000,
+        .addr = 0x000000,
+        .data = {
+                { 0x0080, 0x000000 },
+                { 0x0084, 0x0009a0 },
+                { 0x0088, 0x0009c0 },
+                { 0x008c, 0x0009c8 },
+                { 0x0090, 0x6109b4 },
+                { 0x0094, 0x610970 },
+                { 0x00a0, 0x610998 },
+                { 0x00a4, 0x610964 },
+                { 0x00c0, 0x610958 },
+                { 0x00e0, 0x6109a8 },
+                { 0x00e4, 0x6109d0 },
+                { 0x00e8, 0x6109d8 },
+                { 0x0100, 0x61094c },
+                { 0x0104, 0x610984 },
+                { 0x0108, 0x61098c },
+                { 0x0800, 0x6109f8 },
+                { 0x0808, 0x610a08 },
+                { 0x080c, 0x610a10 },
+                { 0x0810, 0x610a00 },
+                {}
+        }
+};
+
+static const struct nv50_disp_mthd_chan
+nv50_disp_ovly_mthd_chan = {
+        .name = "Overlay",
+        .addr = 0x000540,
+        .data = {
+                { "Global", 1, &nv50_disp_ovly_mthd_base },
+                {}
+        }
+};
+
 static int
 nv50_disp_ovly_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
@@ -782,25 +1040,78 @@ nv50_disp_cclass = {
  * Display engine implementation
  ******************************************************************************/
 
-static void
-nv50_disp_intr_error(struct nv50_disp_priv *priv)
-{
-        u32 channels = (nv_rd32(priv, 0x610020) & 0x001f0000) >> 16;
-        u32 addr, data;
-        int chid;
-
-        for (chid = 0; chid < 5; chid++) {
-                if (!(channels & (1 << chid)))
-                        continue;
+static const struct nouveau_enum
+nv50_disp_intr_error_type[] = {
+        { 3, "ILLEGAL_MTHD" },
+        { 4, "INVALID_VALUE" },
+        { 5, "INVALID_STATE" },
+        { 7, "INVALID_HANDLE" },
+        {}
+};
 
-                nv_wr32(priv, 0x610020, 0x00010000 << chid);
-                addr = nv_rd32(priv, 0x610080 + (chid * 0x08));
-                data = nv_rd32(priv, 0x610084 + (chid * 0x08));
-                nv_wr32(priv, 0x610080 + (chid * 0x08), 0x90000000);
+static const struct nouveau_enum
+nv50_disp_intr_error_code[] = {
+        { 0x00, "" },
+        {}
+};
 
-                nv_error(priv, "chid %d mthd 0x%04x data 0x%08x 0x%08x\n",
-                         chid, addr & 0xffc, data, addr);
+static void
+nv50_disp_intr_error(struct nv50_disp_priv *priv, int chid)
+{
+        struct nv50_disp_impl *impl = (void *)nv_object(priv)->oclass;
+        u32 data = nv_rd32(priv, 0x610084 + (chid * 0x08));
+        u32 addr = nv_rd32(priv, 0x610080 + (chid * 0x08));
+        u32 code = (addr & 0x00ff0000) >> 16;
+        u32 type = (addr & 0x00007000) >> 12;
+        u32 mthd = (addr & 0x00000ffc);
+        const struct nouveau_enum *ec, *et;
+        char ecunk[6], etunk[6];
+
+        et = nouveau_enum_find(nv50_disp_intr_error_type, type);
+        if (!et)
+                snprintf(etunk, sizeof(etunk), "UNK%02X", type);
+
+        ec = nouveau_enum_find(nv50_disp_intr_error_code, code);
+        if (!ec)
+                snprintf(ecunk, sizeof(ecunk), "UNK%02X", code);
+
+        nv_error(priv, "%s [%s] chid %d mthd 0x%04x data 0x%08x\n",
+                 et ? et->name : etunk, ec ? ec->name : ecunk,
+                 chid, mthd, data);
+
+        if (chid == 0) {
+                switch (mthd) {
+                case 0x0080:
+                        nv50_disp_mthd_chan(priv, NV_DBG_ERROR, chid - 0,
+                                            impl->mthd.core);
+                        break;
+                default:
+                        break;
+                }
+        } else
+        if (chid <= 2) {
+                switch (mthd) {
+                case 0x0080:
+                        nv50_disp_mthd_chan(priv, NV_DBG_ERROR, chid - 1,
+                                            impl->mthd.base);
+                        break;
+                default:
+                        break;
+                }
+        } else
+        if (chid <= 4) {
+                switch (mthd) {
+                case 0x0080:
+                        nv50_disp_mthd_chan(priv, NV_DBG_ERROR, chid - 3,
+                                            impl->mthd.ovly);
+                        break;
+                default:
+                        break;
+                }
         }
+
+        nv_wr32(priv, 0x610020, 0x00010000 << chid);
+        nv_wr32(priv, 0x610080 + (chid * 0x08), 0x90000000);
 }
 
 static u16
@@ -1241,12 +1552,14 @@ nv50_disp_intr_supervisor(struct work_struct *work)
 {
         struct nv50_disp_priv *priv =
                 container_of(work, struct nv50_disp_priv, supervisor);
+        struct nv50_disp_impl *impl = (void *)nv_object(priv)->oclass;
         u32 super = nv_rd32(priv, 0x610030);
         int head;
 
         nv_debug(priv, "supervisor 0x%08x 0x%08x\n", priv->super, super);
 
         if (priv->super & 0x00000010) {
+                nv50_disp_mthd_chan(priv, NV_DBG_DEBUG, 0, impl->mthd.core);
                 for (head = 0; head < priv->head.nr; head++) {
                         if (!(super & (0x00000020 << head)))
                                 continue;
@@ -1290,9 +1603,10 @@ nv50_disp_intr(struct nouveau_subdev *subdev)
         u32 intr0 = nv_rd32(priv, 0x610020);
         u32 intr1 = nv_rd32(priv, 0x610024);
 
-        if (intr0 & 0x001f0000) {
-                nv50_disp_intr_error(priv);
-                intr0 &= ~0x001f0000;
+        while (intr0 & 0x001f0000) {
+                u32 chid = __ffs(intr0 & 0x001f0000) - 16;
+                nv50_disp_intr_error(priv, chid);
+                intr0 &= ~(0x00010000 << chid);
         }
 
         if (intr1 & 0x00000004) {
@@ -1346,13 +1660,17 @@ nv50_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-struct nouveau_oclass
-nv50_disp_oclass = {
-        .handle = NV_ENGINE(DISP, 0x50),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nv50_disp_oclass = &(struct nv50_disp_impl) {
+        .base.base.handle = NV_ENGINE(DISP, 0x50),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv50_disp_ctor,
                 .dtor = _nouveau_disp_dtor,
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
-};
+        .mthd.core = &nv50_disp_mast_mthd_chan,
+        .mthd.base = &nv50_disp_sync_mthd_chan,
+        .mthd.ovly = &nv50_disp_ovly_mthd_chan,
+        .mthd.prev = 0x000004,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h
index d31d426ea1f6..48d59db47f0d 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv50.h
@@ -8,9 +8,19 @@
 #include <core/event.h>
 
 #include <engine/dmaobj.h>
-#include <engine/disp.h>
 
 #include "dport.h"
+#include "priv.h"
+
+struct nv50_disp_impl {
+        struct nouveau_disp_impl base;
+        struct {
+                const struct nv50_disp_mthd_chan *core;
+                const struct nv50_disp_mthd_chan *base;
+                const struct nv50_disp_mthd_chan *ovly;
+                int prev;
+        } mthd;
+};
 
 struct nv50_disp_priv {
         struct nouveau_disp base;
@@ -124,21 +134,60 @@ struct nv50_disp_pioc {
         struct nv50_disp_chan base;
 };
 
+struct nv50_disp_mthd_list {
+        u32 mthd;
+        u32 addr;
+        struct {
+                u32 mthd;
+                u32 addr;
+                const char *name;
+        } data[];
+};
+
+struct nv50_disp_mthd_chan {
+        const char *name;
+        u32 addr;
+        struct {
+                const char *name;
+                int nr;
+                const struct nv50_disp_mthd_list *mthd;
+        } data[];
+};
+
 extern struct nouveau_ofuncs nv50_disp_mast_ofuncs;
+extern const struct nv50_disp_mthd_list nv50_disp_mast_mthd_base;
+extern const struct nv50_disp_mthd_list nv50_disp_mast_mthd_sor;
+extern const struct nv50_disp_mthd_list nv50_disp_mast_mthd_pior;
 extern struct nouveau_ofuncs nv50_disp_sync_ofuncs;
+extern const struct nv50_disp_mthd_list nv50_disp_sync_mthd_image;
 extern struct nouveau_ofuncs nv50_disp_ovly_ofuncs;
+extern const struct nv50_disp_mthd_list nv50_disp_ovly_mthd_base;
 extern struct nouveau_ofuncs nv50_disp_oimm_ofuncs;
 extern struct nouveau_ofuncs nv50_disp_curs_ofuncs;
 extern struct nouveau_ofuncs nv50_disp_base_ofuncs;
 extern struct nouveau_oclass nv50_disp_cclass;
+void nv50_disp_mthd_chan(struct nv50_disp_priv *, int debug, int head,
+                         const struct nv50_disp_mthd_chan *);
 void nv50_disp_intr_supervisor(struct work_struct *);
 void nv50_disp_intr(struct nouveau_subdev *);
 
+extern const struct nv50_disp_mthd_chan nv84_disp_mast_mthd_chan;
+extern const struct nv50_disp_mthd_list nv84_disp_mast_mthd_dac;
+extern const struct nv50_disp_mthd_list nv84_disp_mast_mthd_head;
+extern const struct nv50_disp_mthd_chan nv84_disp_sync_mthd_chan;
+extern const struct nv50_disp_mthd_chan nv84_disp_ovly_mthd_chan;
 extern struct nouveau_omthds nv84_disp_base_omthds[];
 
+extern const struct nv50_disp_mthd_chan nv94_disp_mast_mthd_chan;
+
 extern struct nouveau_ofuncs nvd0_disp_mast_ofuncs;
+extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_base;
+extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_dac;
+extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_sor;
+extern const struct nv50_disp_mthd_list nvd0_disp_mast_mthd_pior;
 extern struct nouveau_ofuncs nvd0_disp_sync_ofuncs;
 extern struct nouveau_ofuncs nvd0_disp_ovly_ofuncs;
+extern const struct nv50_disp_mthd_chan nvd0_disp_sync_mthd_chan;
 extern struct nouveau_ofuncs nvd0_disp_oimm_ofuncs;
 extern struct nouveau_ofuncs nvd0_disp_curs_ofuncs;
 extern struct nouveau_omthds nvd0_disp_base_omthds[];
@@ -147,4 +196,7 @@ extern struct nouveau_oclass nvd0_disp_cclass;
 void nvd0_disp_intr_supervisor(struct work_struct *);
 void nvd0_disp_intr(struct nouveau_subdev *);
 
+extern const struct nv50_disp_mthd_chan nve0_disp_mast_mthd_chan;
+extern const struct nv50_disp_mthd_chan nve0_disp_ovly_mthd_chan;
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c
index ef9ce300a496..98c5b19bc2b0 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv84.c
@@ -29,6 +29,179 @@
 
 #include "nv50.h"
 
+/*******************************************************************************
+ * EVO master channel object
+ ******************************************************************************/
+
+const struct nv50_disp_mthd_list
+nv84_disp_mast_mthd_dac = {
+        .mthd = 0x0080,
+        .addr = 0x000008,
+        .data = {
+                { 0x0400, 0x610b58 },
+                { 0x0404, 0x610bdc },
+                { 0x0420, 0x610bc4 },
+                {}
+        }
+};
+
+const struct nv50_disp_mthd_list
+nv84_disp_mast_mthd_head = {
+        .mthd = 0x0400,
+        .addr = 0x000540,
+        .data = {
+                { 0x0800, 0x610ad8 },
+                { 0x0804, 0x610ad0 },
+                { 0x0808, 0x610a48 },
+                { 0x080c, 0x610a78 },
+                { 0x0810, 0x610ac0 },
+                { 0x0814, 0x610af8 },
+                { 0x0818, 0x610b00 },
+                { 0x081c, 0x610ae8 },
+                { 0x0820, 0x610af0 },
+                { 0x0824, 0x610b08 },
+                { 0x0828, 0x610b10 },
+                { 0x082c, 0x610a68 },
+                { 0x0830, 0x610a60 },
+                { 0x0834, 0x000000 },
+                { 0x0838, 0x610a40 },
+                { 0x0840, 0x610a24 },
+                { 0x0844, 0x610a2c },
+                { 0x0848, 0x610aa8 },
+                { 0x084c, 0x610ab0 },
+                { 0x085c, 0x610c5c },
+                { 0x0860, 0x610a84 },
+                { 0x0864, 0x610a90 },
+                { 0x0868, 0x610b18 },
+                { 0x086c, 0x610b20 },
+                { 0x0870, 0x610ac8 },
+                { 0x0874, 0x610a38 },
+                { 0x0878, 0x610c50 },
+                { 0x0880, 0x610a58 },
+                { 0x0884, 0x610a9c },
+                { 0x089c, 0x610c68 },
+                { 0x08a0, 0x610a70 },
+                { 0x08a4, 0x610a50 },
+                { 0x08a8, 0x610ae0 },
+                { 0x08c0, 0x610b28 },
+                { 0x08c4, 0x610b30 },
+                { 0x08c8, 0x610b40 },
+                { 0x08d4, 0x610b38 },
+                { 0x08d8, 0x610b48 },
+                { 0x08dc, 0x610b50 },
+                { 0x0900, 0x610a18 },
+                { 0x0904, 0x610ab8 },
+                { 0x0910, 0x610c70 },
+                { 0x0914, 0x610c78 },
+                {}
+        }
+};
+
+const struct nv50_disp_mthd_chan
+nv84_disp_mast_mthd_chan = {
+        .name = "Core",
+        .addr = 0x000000,
+        .data = {
+                { "Global", 1, &nv50_disp_mast_mthd_base },
+                {    "DAC", 3, &nv84_disp_mast_mthd_dac  },
+                {    "SOR", 2, &nv50_disp_mast_mthd_sor  },
+                {   "PIOR", 3, &nv50_disp_mast_mthd_pior },
+                {   "HEAD", 2, &nv84_disp_mast_mthd_head },
+                {}
+        }
+};
+
+/*******************************************************************************
+ * EVO sync channel objects
+ ******************************************************************************/
+
+static const struct nv50_disp_mthd_list
+nv84_disp_sync_mthd_base = {
+        .mthd = 0x0000,
+        .addr = 0x000000,
+        .data = {
+                { 0x0080, 0x000000 },
+                { 0x0084, 0x0008c4 },
+                { 0x0088, 0x0008d0 },
+                { 0x008c, 0x0008dc },
+                { 0x0090, 0x0008e4 },
+                { 0x0094, 0x610884 },
+                { 0x00a0, 0x6108a0 },
+                { 0x00a4, 0x610878 },
+                { 0x00c0, 0x61086c },
+                { 0x00c4, 0x610800 },
+                { 0x00c8, 0x61080c },
+                { 0x00cc, 0x610818 },
+                { 0x00e0, 0x610858 },
+                { 0x00e4, 0x610860 },
+                { 0x00e8, 0x6108ac },
+                { 0x00ec, 0x6108b4 },
+                { 0x00fc, 0x610824 },
+                { 0x0100, 0x610894 },
+                { 0x0104, 0x61082c },
+                { 0x0110, 0x6108bc },
+                { 0x0114, 0x61088c },
+                {}
+        }
+};
+
+const struct nv50_disp_mthd_chan
+nv84_disp_sync_mthd_chan = {
+        .name = "Base",
+        .addr = 0x000540,
+        .data = {
+                { "Global", 1, &nv84_disp_sync_mthd_base },
+                {  "Image", 2, &nv50_disp_sync_mthd_image },
+                {}
+        }
+};
+
+/*******************************************************************************
+ * EVO overlay channel objects
+ ******************************************************************************/
+
+static const struct nv50_disp_mthd_list
+nv84_disp_ovly_mthd_base = {
+        .mthd = 0x0000,
+        .addr = 0x000000,
+        .data = {
+                { 0x0080, 0x000000 },
+                { 0x0084, 0x6109a0 },
+                { 0x0088, 0x6109c0 },
+                { 0x008c, 0x6109c8 },
+                { 0x0090, 0x6109b4 },
+                { 0x0094, 0x610970 },
+                { 0x00a0, 0x610998 },
+                { 0x00a4, 0x610964 },
+                { 0x00c0, 0x610958 },
+                { 0x00e0, 0x6109a8 },
+                { 0x00e4, 0x6109d0 },
+                { 0x00e8, 0x6109d8 },
+                { 0x0100, 0x61094c },
+                { 0x0104, 0x610984 },
+                { 0x0108, 0x61098c },
+                { 0x0800, 0x6109f8 },
+                { 0x0808, 0x610a08 },
+                { 0x080c, 0x610a10 },
+                { 0x0810, 0x610a00 },
+                {}
+        }
+};
+
+const struct nv50_disp_mthd_chan
+nv84_disp_ovly_mthd_chan = {
+        .name = "Overlay",
+        .addr = 0x000540,
+        .data = {
+                { "Global", 1, &nv84_disp_ovly_mthd_base },
+                {}
+        }
+};
+
+/*******************************************************************************
+ * Base display object
+ ******************************************************************************/
+
 static struct nouveau_oclass
 nv84_disp_sclass[] = {
         { NV84_DISP_MAST_CLASS, &nv50_disp_mast_ofuncs },
@@ -59,6 +232,10 @@ nv84_disp_base_oclass[] = {
         {}
 };
 
+/*******************************************************************************
+ * Display engine implementation
+ ******************************************************************************/
+
 static int
 nv84_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                struct nouveau_oclass *oclass, void *data, u32 size,
@@ -91,13 +268,17 @@ nv84_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-struct nouveau_oclass
-nv84_disp_oclass = {
-        .handle = NV_ENGINE(DISP, 0x82),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nv84_disp_oclass = &(struct nv50_disp_impl) {
+        .base.base.handle = NV_ENGINE(DISP, 0x82),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv84_disp_ctor,
                 .dtor = _nouveau_disp_dtor,
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
-};
+        .mthd.core = &nv84_disp_mast_mthd_chan,
+        .mthd.base = &nv84_disp_sync_mthd_chan,
+        .mthd.ovly = &nv84_disp_ovly_mthd_chan,
+        .mthd.prev = 0x000004,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c b/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c
index a518543c00ab..6844061c7e04 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nv94.c
@@ -29,6 +29,38 @@
 
 #include "nv50.h"
 
+/*******************************************************************************
+ * EVO master channel object
+ ******************************************************************************/
+
+const struct nv50_disp_mthd_list
+nv94_disp_mast_mthd_sor = {
+        .mthd = 0x0040,
+        .addr = 0x000008,
+        .data = {
+                { 0x0600, 0x610794 },
+                {}
+        }
+};
+
+const struct nv50_disp_mthd_chan
+nv94_disp_mast_mthd_chan = {
+        .name = "Core",
+        .addr = 0x000000,
+        .data = {
+                { "Global", 1, &nv50_disp_mast_mthd_base },
+                {    "DAC", 3, &nv84_disp_mast_mthd_dac  },
+                {    "SOR", 4, &nv94_disp_mast_mthd_sor  },
+                {   "PIOR", 3, &nv50_disp_mast_mthd_pior },
+                {   "HEAD", 2, &nv84_disp_mast_mthd_head },
+                {}
+        }
+};
+
+/*******************************************************************************
+ * Base display object
+ ******************************************************************************/
+
 static struct nouveau_oclass
 nv94_disp_sclass[] = {
         { NV94_DISP_MAST_CLASS, &nv50_disp_mast_ofuncs },
@@ -59,6 +91,10 @@ nv94_disp_base_oclass[] = {
         {}
 };
 
+/*******************************************************************************
+ * Display engine implementation
+ ******************************************************************************/
+
 static int
 nv94_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                struct nouveau_oclass *oclass, void *data, u32 size,
@@ -92,13 +128,17 @@ nv94_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-struct nouveau_oclass
-nv94_disp_oclass = {
-        .handle = NV_ENGINE(DISP, 0x88),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nv94_disp_oclass = &(struct nv50_disp_impl) {
+        .base.base.handle = NV_ENGINE(DISP, 0x88),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nv94_disp_ctor,
                 .dtor = _nouveau_disp_dtor,
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
-};
+        .mthd.core = &nv94_disp_mast_mthd_chan,
+        .mthd.base = &nv84_disp_sync_mthd_chan,
+        .mthd.ovly = &nv84_disp_ovly_mthd_chan,
+        .mthd.prev = 0x000004,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c
index 6cf8eefac368..88c96241c02a 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nva0.c
@@ -29,6 +29,55 @@
 
 #include "nv50.h"
 
+/*******************************************************************************
+ * EVO overlay channel objects
+ ******************************************************************************/
+
+static const struct nv50_disp_mthd_list
+nva0_disp_ovly_mthd_base = {
+        .mthd = 0x0000,
+        .addr = 0x000000,
+        .data = {
+                { 0x0080, 0x000000 },
+                { 0x0084, 0x6109a0 },
+                { 0x0088, 0x6109c0 },
+                { 0x008c, 0x6109c8 },
+                { 0x0090, 0x6109b4 },
+                { 0x0094, 0x610970 },
+                { 0x00a0, 0x610998 },
+                { 0x00a4, 0x610964 },
+                { 0x00b0, 0x610c98 },
+                { 0x00b4, 0x610ca4 },
+                { 0x00b8, 0x610cac },
+                { 0x00c0, 0x610958 },
+                { 0x00e0, 0x6109a8 },
+                { 0x00e4, 0x6109d0 },
+                { 0x00e8, 0x6109d8 },
+                { 0x0100, 0x61094c },
+                { 0x0104, 0x610984 },
+                { 0x0108, 0x61098c },
+                { 0x0800, 0x6109f8 },
+                { 0x0808, 0x610a08 },
+                { 0x080c, 0x610a10 },
+                { 0x0810, 0x610a00 },
+                {}
+        }
+};
+
+static const struct nv50_disp_mthd_chan
+nva0_disp_ovly_mthd_chan = {
+        .name = "Overlay",
+        .addr = 0x000540,
+        .data = {
+                { "Global", 1, &nva0_disp_ovly_mthd_base },
+                {}
+        }
+};
+
+/*******************************************************************************
+ * Base display object
+ ******************************************************************************/
+
 static struct nouveau_oclass
 nva0_disp_sclass[] = {
         { NVA0_DISP_MAST_CLASS, &nv50_disp_mast_ofuncs },
@@ -45,6 +94,10 @@ nva0_disp_base_oclass[] = {
         {}
 };
 
+/*******************************************************************************
+ * Display engine implementation
+ ******************************************************************************/
+
 static int
 nva0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                struct nouveau_oclass *oclass, void *data, u32 size,
@@ -77,13 +130,17 @@ nva0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-struct nouveau_oclass
-nva0_disp_oclass = {
-        .handle = NV_ENGINE(DISP, 0x83),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nva0_disp_oclass = &(struct nv50_disp_impl) {
+        .base.base.handle = NV_ENGINE(DISP, 0x83),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nva0_disp_ctor,
                 .dtor = _nouveau_disp_dtor,
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
-};
+        .mthd.core = &nv84_disp_mast_mthd_chan,
+        .mthd.base = &nv84_disp_sync_mthd_chan,
+        .mthd.ovly = &nva0_disp_ovly_mthd_chan,
+        .mthd.prev = 0x000004,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c b/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c
index 6ad6dcece43b..46cb2ce0e82a 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nva3.c
@@ -29,6 +29,10 @@
 
 #include "nv50.h"
 
+/*******************************************************************************
+ * Base display object
+ ******************************************************************************/
+
 static struct nouveau_oclass
 nva3_disp_sclass[] = {
         { NVA3_DISP_MAST_CLASS, &nv50_disp_mast_ofuncs },
@@ -60,6 +64,10 @@ nva3_disp_base_oclass[] = {
         {}
 };
 
+/*******************************************************************************
+ * Display engine implementation
+ ******************************************************************************/
+
 static int
 nva3_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                struct nouveau_oclass *oclass, void *data, u32 size,
@@ -94,13 +102,17 @@ nva3_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-struct nouveau_oclass
-nva3_disp_oclass = {
-        .handle = NV_ENGINE(DISP, 0x85),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nva3_disp_oclass = &(struct nv50_disp_impl) {
+        .base.base.handle = NV_ENGINE(DISP, 0x85),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nva3_disp_ctor,
                 .dtor = _nouveau_disp_dtor,
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
-};
+        .mthd.core = &nv94_disp_mast_mthd_chan,
+        .mthd.base = &nv84_disp_sync_mthd_chan,
+        .mthd.ovly = &nv84_disp_ovly_mthd_chan,
+        .mthd.prev = 0x000004,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
index 1c5e4e8b2c82..7762665ad8fd 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nvd0.c
@@ -124,6 +124,146 @@ nvd0_disp_dmac_fini(struct nouveau_object *object, bool suspend)
  * EVO master channel object
  ******************************************************************************/
 
+const struct nv50_disp_mthd_list
+nvd0_disp_mast_mthd_base = {
+        .mthd = 0x0000,
+        .addr = 0x000000,
+        .data = {
+                { 0x0080, 0x660080 },
+                { 0x0084, 0x660084 },
+                { 0x0088, 0x660088 },
+                { 0x008c, 0x000000 },
+                {}
+        }
+};
+
+const struct nv50_disp_mthd_list
+nvd0_disp_mast_mthd_dac = {
+        .mthd = 0x0020,
+        .addr = 0x000020,
+        .data = {
+                { 0x0180, 0x660180 },
+                { 0x0184, 0x660184 },
+                { 0x0188, 0x660188 },
+                { 0x0190, 0x660190 },
+                {}
+        }
+};
+
+const struct nv50_disp_mthd_list
+nvd0_disp_mast_mthd_sor = {
+        .mthd = 0x0020,
+        .addr = 0x000020,
+        .data = {
+                { 0x0200, 0x660200 },
+                { 0x0204, 0x660204 },
+                { 0x0208, 0x660208 },
+                { 0x0210, 0x660210 },
+                {}
+        }
+};
+
+const struct nv50_disp_mthd_list
+nvd0_disp_mast_mthd_pior = {
+        .mthd = 0x0020,
+        .addr = 0x000020,
+        .data = {
+                { 0x0300, 0x660300 },
+                { 0x0304, 0x660304 },
+                { 0x0308, 0x660308 },
+                { 0x0310, 0x660310 },
+                {}
+        }
+};
+
+static const struct nv50_disp_mthd_list
+nvd0_disp_mast_mthd_head = {
+        .mthd = 0x0300,
+        .addr = 0x000300,
+        .data = {
+                { 0x0400, 0x660400 },
+                { 0x0404, 0x660404 },
+                { 0x0408, 0x660408 },
+                { 0x040c, 0x66040c },
+                { 0x0410, 0x660410 },
+                { 0x0414, 0x660414 },
+                { 0x0418, 0x660418 },
+                { 0x041c, 0x66041c },
+                { 0x0420, 0x660420 },
+                { 0x0424, 0x660424 },
+                { 0x0428, 0x660428 },
+                { 0x042c, 0x66042c },
+                { 0x0430, 0x660430 },
+                { 0x0434, 0x660434 },
+                { 0x0438, 0x660438 },
+                { 0x0440, 0x660440 },
+                { 0x0444, 0x660444 },
+                { 0x0448, 0x660448 },
+                { 0x044c, 0x66044c },
+                { 0x0450, 0x660450 },
+                { 0x0454, 0x660454 },
+                { 0x0458, 0x660458 },
+                { 0x045c, 0x66045c },
+                { 0x0460, 0x660460 },
+                { 0x0468, 0x660468 },
+                { 0x046c, 0x66046c },
+                { 0x0470, 0x660470 },
+                { 0x0474, 0x660474 },
+                { 0x0480, 0x660480 },
+                { 0x0484, 0x660484 },
+                { 0x048c, 0x66048c },
+                { 0x0490, 0x660490 },
+                { 0x0494, 0x660494 },
+                { 0x0498, 0x660498 },
+                { 0x04b0, 0x6604b0 },
+                { 0x04b8, 0x6604b8 },
+                { 0x04bc, 0x6604bc },
+                { 0x04c0, 0x6604c0 },
+                { 0x04c4, 0x6604c4 },
+                { 0x04c8, 0x6604c8 },
+                { 0x04d0, 0x6604d0 },
+                { 0x04d4, 0x6604d4 },
+                { 0x04e0, 0x6604e0 },
+                { 0x04e4, 0x6604e4 },
+                { 0x04e8, 0x6604e8 },
+                { 0x04ec, 0x6604ec },
+                { 0x04f0, 0x6604f0 },
+                { 0x04f4, 0x6604f4 },
+                { 0x04f8, 0x6604f8 },
+                { 0x04fc, 0x6604fc },
+                { 0x0500, 0x660500 },
+                { 0x0504, 0x660504 },
+                { 0x0508, 0x660508 },
+                { 0x050c, 0x66050c },
+                { 0x0510, 0x660510 },
+                { 0x0514, 0x660514 },
+                { 0x0518, 0x660518 },
+                { 0x051c, 0x66051c },
+                { 0x052c, 0x66052c },
+                { 0x0530, 0x660530 },
+                { 0x054c, 0x66054c },
+                { 0x0550, 0x660550 },
+                { 0x0554, 0x660554 },
+                { 0x0558, 0x660558 },
+                { 0x055c, 0x66055c },
+                {}
+        }
+};
+
+static const struct nv50_disp_mthd_chan
+nvd0_disp_mast_mthd_chan = {
+        .name = "Core",
+        .addr = 0x000000,
+        .data = {
+                { "Global", 1, &nvd0_disp_mast_mthd_base },
+                {    "DAC", 3, &nvd0_disp_mast_mthd_dac  },
+                {    "SOR", 8, &nvd0_disp_mast_mthd_sor  },
+                {   "PIOR", 4, &nvd0_disp_mast_mthd_pior },
+                {   "HEAD", 4, &nvd0_disp_mast_mthd_head },
+                {}
+        }
+};
+
 static int
 nvd0_disp_mast_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
@@ -216,6 +356,81 @@ nvd0_disp_mast_ofuncs = {
  * EVO sync channel objects
  ******************************************************************************/
 
+static const struct nv50_disp_mthd_list
+nvd0_disp_sync_mthd_base = {
+        .mthd = 0x0000,
+        .addr = 0x000000,
+        .data = {
+                { 0x0080, 0x661080 },
+                { 0x0084, 0x661084 },
+                { 0x0088, 0x661088 },
+                { 0x008c, 0x66108c },
+                { 0x0090, 0x661090 },
+                { 0x0094, 0x661094 },
+                { 0x00a0, 0x6610a0 },
+                { 0x00a4, 0x6610a4 },
+                { 0x00c0, 0x6610c0 },
+                { 0x00c4, 0x6610c4 },
+                { 0x00c8, 0x6610c8 },
+                { 0x00cc, 0x6610cc },
+                { 0x00e0, 0x6610e0 },
+                { 0x00e4, 0x6610e4 },
+                { 0x00e8, 0x6610e8 },
+                { 0x00ec, 0x6610ec },
+                { 0x00fc, 0x6610fc },
+                { 0x0100, 0x661100 },
+                { 0x0104, 0x661104 },
+                { 0x0108, 0x661108 },
+                { 0x010c, 0x66110c },
+                { 0x0110, 0x661110 },
+                { 0x0114, 0x661114 },
+                { 0x0118, 0x661118 },
+                { 0x011c, 0x66111c },
+                { 0x0130, 0x661130 },
+                { 0x0134, 0x661134 },
+                { 0x0138, 0x661138 },
+                { 0x013c, 0x66113c },
+                { 0x0140, 0x661140 },
+                { 0x0144, 0x661144 },
+                { 0x0148, 0x661148 },
+                { 0x014c, 0x66114c },
+                { 0x0150, 0x661150 },
+                { 0x0154, 0x661154 },
+                { 0x0158, 0x661158 },
+                { 0x015c, 0x66115c },
+                { 0x0160, 0x661160 },
+                { 0x0164, 0x661164 },
+                { 0x0168, 0x661168 },
+                { 0x016c, 0x66116c },
+                {}
+        }
+};
+
+static const struct nv50_disp_mthd_list
+nvd0_disp_sync_mthd_image = {
+        .mthd = 0x0400,
+        .addr = 0x000400,
+        .data = {
+                { 0x0400, 0x661400 },
+                { 0x0404, 0x661404 },
+                { 0x0408, 0x661408 },
+                { 0x040c, 0x66140c },
+                { 0x0410, 0x661410 },
+                {}
+        }
+};
+
+const struct nv50_disp_mthd_chan
+nvd0_disp_sync_mthd_chan = {
+        .name = "Base",
+        .addr = 0x001000,
+        .data = {
+                { "Global", 1, &nvd0_disp_sync_mthd_base },
+                {  "Image", 2, &nvd0_disp_sync_mthd_image },
+                {}
+        }
+};
+
 static int
 nvd0_disp_sync_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
@@ -256,6 +471,68 @@ nvd0_disp_sync_ofuncs = {
  * EVO overlay channel objects
  ******************************************************************************/
 
+static const struct nv50_disp_mthd_list
+nvd0_disp_ovly_mthd_base = {
+        .mthd = 0x0000,
+        .data = {
+                { 0x0080, 0x665080 },
+                { 0x0084, 0x665084 },
+                { 0x0088, 0x665088 },
+                { 0x008c, 0x66508c },
+                { 0x0090, 0x665090 },
+                { 0x0094, 0x665094 },
+                { 0x00a0, 0x6650a0 },
+                { 0x00a4, 0x6650a4 },
+                { 0x00b0, 0x6650b0 },
+                { 0x00b4, 0x6650b4 },
+                { 0x00b8, 0x6650b8 },
+                { 0x00c0, 0x6650c0 },
+                { 0x00e0, 0x6650e0 },
+                { 0x00e4, 0x6650e4 },
+                { 0x00e8, 0x6650e8 },
+                { 0x0100, 0x665100 },
+                { 0x0104, 0x665104 },
+                { 0x0108, 0x665108 },
+                { 0x010c, 0x66510c },
+                { 0x0110, 0x665110 },
+                { 0x0118, 0x665118 },
+                { 0x011c, 0x66511c },
+                { 0x0120, 0x665120 },
+                { 0x0124, 0x665124 },
+                { 0x0130, 0x665130 },
+                { 0x0134, 0x665134 },
+                { 0x0138, 0x665138 },
+                { 0x013c, 0x66513c },
+                { 0x0140, 0x665140 },
+                { 0x0144, 0x665144 },
+                { 0x0148, 0x665148 },
+                { 0x014c, 0x66514c },
+                { 0x0150, 0x665150 },
+                { 0x0154, 0x665154 },
+                { 0x0158, 0x665158 },
+                { 0x015c, 0x66515c },
+                { 0x0160, 0x665160 },
+                { 0x0164, 0x665164 },
+                { 0x0168, 0x665168 },
+                { 0x016c, 0x66516c },
+                { 0x0400, 0x665400 },
+                { 0x0408, 0x665408 },
+                { 0x040c, 0x66540c },
+                { 0x0410, 0x665410 },
+                {}
+        }
+};
+
+static const struct nv50_disp_mthd_chan
+nvd0_disp_ovly_mthd_chan = {
+        .name = "Overlay",
+        .addr = 0x001000,
+        .data = {
+                { "Global", 1, &nvd0_disp_ovly_mthd_base },
+                {}
+        }
+};
+
 static int
 nvd0_disp_ovly_ctor(struct nouveau_object *parent,
                     struct nouveau_object *engine,
@@ -897,19 +1174,22 @@ nvd0_disp_intr_supervisor(struct work_struct *work)
 {
         struct nv50_disp_priv *priv =
                 container_of(work, struct nv50_disp_priv, supervisor);
+        struct nv50_disp_impl *impl = (void *)nv_object(priv)->oclass;
         u32 mask[4];
         int head;
 
-        nv_debug(priv, "supervisor %08x\n", priv->super);
+        nv_debug(priv, "supervisor %d\n", ffs(priv->super));
         for (head = 0; head < priv->head.nr; head++) {
                 mask[head] = nv_rd32(priv, 0x6101d4 + (head * 0x800));
                 nv_debug(priv, "head %d: 0x%08x\n", head, mask[head]);
         }
 
         if (priv->super & 0x00000001) {
+                nv50_disp_mthd_chan(priv, NV_DBG_DEBUG, 0, impl->mthd.core);
                 for (head = 0; head < priv->head.nr; head++) {
                         if (!(mask[head] & 0x00001000))
                                 continue;
+                        nv_debug(priv, "supervisor 1.0 - head %d\n", head);
                         nvd0_disp_intr_unk1_0(priv, head);
                 }
         } else
@@ -917,16 +1197,19 @@ nvd0_disp_intr_supervisor(struct work_struct *work)
                 for (head = 0; head < priv->head.nr; head++) {
                         if (!(mask[head] & 0x00001000))
                                 continue;
+                        nv_debug(priv, "supervisor 2.0 - head %d\n", head);
                         nvd0_disp_intr_unk2_0(priv, head);
                 }
                 for (head = 0; head < priv->head.nr; head++) {
                         if (!(mask[head] & 0x00010000))
                                 continue;
+                        nv_debug(priv, "supervisor 2.1 - head %d\n", head);
                         nvd0_disp_intr_unk2_1(priv, head);
                 }
                 for (head = 0; head < priv->head.nr; head++) {
                         if (!(mask[head] & 0x00001000))
                                 continue;
+                        nv_debug(priv, "supervisor 2.2 - head %d\n", head);
                         nvd0_disp_intr_unk2_2(priv, head);
                 }
         } else
@@ -934,6 +1217,7 @@ nvd0_disp_intr_supervisor(struct work_struct *work)
                 for (head = 0; head < priv->head.nr; head++) {
                         if (!(mask[head] & 0x00001000))
                                 continue;
+                        nv_debug(priv, "supervisor 3.0 - head %d\n", head);
                         nvd0_disp_intr_unk4_0(priv, head);
                 }
         }
@@ -943,6 +1227,53 @@ nvd0_disp_intr_supervisor(struct work_struct *work)
         nv_wr32(priv, 0x6101d0, 0x80000000);
 }
 
+static void
+nvd0_disp_intr_error(struct nv50_disp_priv *priv, int chid)
+{
+        const struct nv50_disp_impl *impl = (void *)nv_object(priv)->oclass;
+        u32 mthd = nv_rd32(priv, 0x6101f0 + (chid * 12));
+        u32 data = nv_rd32(priv, 0x6101f4 + (chid * 12));
+        u32 unkn = nv_rd32(priv, 0x6101f8 + (chid * 12));
+
+        nv_error(priv, "chid %d mthd 0x%04x data 0x%08x "
+                       "0x%08x 0x%08x\n",
+                 chid, (mthd & 0x0000ffc), data, mthd, unkn);
+
+        if (chid == 0) {
+                switch (mthd) {
+                case 0x0080:
+                        nv50_disp_mthd_chan(priv, NV_DBG_ERROR, chid - 0,
+                                            impl->mthd.core);
+                        break;
+                default:
+                        break;
+                }
+        } else
+        if (chid <= 4) {
+                switch (mthd) {
+                case 0x0080:
+                        nv50_disp_mthd_chan(priv, NV_DBG_ERROR, chid - 1,
+                                            impl->mthd.base);
+                        break;
+                default:
+                        break;
+                }
+        } else
+        if (chid <= 8) {
+                switch (mthd) {
+                case 0x0080:
+                        nv50_disp_mthd_chan(priv, NV_DBG_ERROR, chid - 5,
+                                            impl->mthd.ovly);
+                        break;
+                default:
+                        break;
+                }
+        }
+
+        nv_wr32(priv, 0x61009c, (1 << chid));
+        nv_wr32(priv, 0x6101f0 + (chid * 12), 0x90000000);
+}
+
 void
 nvd0_disp_intr(struct nouveau_subdev *subdev)
 {
@@ -959,18 +1290,8 @@ nvd0_disp_intr(struct nouveau_subdev *subdev)
         if (intr & 0x00000002) {
                 u32 stat = nv_rd32(priv, 0x61009c);
                 int chid = ffs(stat) - 1;
-                if (chid >= 0) {
-                        u32 mthd = nv_rd32(priv, 0x6101f0 + (chid * 12));
-                        u32 data = nv_rd32(priv, 0x6101f4 + (chid * 12));
-                        u32 unkn = nv_rd32(priv, 0x6101f8 + (chid * 12));
-
-                        nv_error(priv, "chid %d mthd 0x%04x data 0x%08x "
-                                       "0x%08x 0x%08x\n",
-                                 chid, (mthd & 0x0000ffc), data, mthd, unkn);
-                        nv_wr32(priv, 0x61009c, (1 << chid));
-                        nv_wr32(priv, 0x6101f0 + (chid * 12), 0x90000000);
-                }
-
+                if (chid >= 0)
+                        nvd0_disp_intr_error(priv, chid);
                 intr &= ~0x00000002;
         }
 
@@ -1035,13 +1356,17 @@ nvd0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-struct nouveau_oclass
-nvd0_disp_oclass = {
-        .handle = NV_ENGINE(DISP, 0x90),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nvd0_disp_oclass = &(struct nv50_disp_impl) {
+        .base.base.handle = NV_ENGINE(DISP, 0x90),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nvd0_disp_ctor,
                 .dtor = _nouveau_disp_dtor,
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
-};
+        .mthd.core = &nvd0_disp_mast_mthd_chan,
+        .mthd.base = &nvd0_disp_sync_mthd_chan,
+        .mthd.ovly = &nvd0_disp_ovly_mthd_chan,
+        .mthd.prev = -0x020000,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c
index ab63f32c00b2..44e0b8f34c1a 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nve0.c
@@ -29,6 +29,175 @@
 
 #include "nv50.h"
 
+/*******************************************************************************
+ * EVO master channel object
+ ******************************************************************************/
+
+static const struct nv50_disp_mthd_list
+nve0_disp_mast_mthd_head = {
+        .mthd = 0x0300,
+        .addr = 0x000300,
+        .data = {
+                { 0x0400, 0x660400 },
+                { 0x0404, 0x660404 },
+                { 0x0408, 0x660408 },
+                { 0x040c, 0x66040c },
+                { 0x0410, 0x660410 },
+                { 0x0414, 0x660414 },
+                { 0x0418, 0x660418 },
+                { 0x041c, 0x66041c },
+                { 0x0420, 0x660420 },
+                { 0x0424, 0x660424 },
+                { 0x0428, 0x660428 },
+                { 0x042c, 0x66042c },
+                { 0x0430, 0x660430 },
+                { 0x0434, 0x660434 },
+                { 0x0438, 0x660438 },
+                { 0x0440, 0x660440 },
+                { 0x0444, 0x660444 },
+                { 0x0448, 0x660448 },
+                { 0x044c, 0x66044c },
+                { 0x0450, 0x660450 },
+                { 0x0454, 0x660454 },
+                { 0x0458, 0x660458 },
+                { 0x045c, 0x66045c },
+                { 0x0460, 0x660460 },
+                { 0x0468, 0x660468 },
+                { 0x046c, 0x66046c },
+                { 0x0470, 0x660470 },
+                { 0x0474, 0x660474 },
+                { 0x047c, 0x66047c },
+                { 0x0480, 0x660480 },
+                { 0x0484, 0x660484 },
+                { 0x0488, 0x660488 },
+                { 0x048c, 0x66048c },
+                { 0x0490, 0x660490 },
+                { 0x0494, 0x660494 },
+                { 0x0498, 0x660498 },
+                { 0x04a0, 0x6604a0 },
+                { 0x04b0, 0x6604b0 },
+                { 0x04b8, 0x6604b8 },
+                { 0x04bc, 0x6604bc },
+                { 0x04c0, 0x6604c0 },
+                { 0x04c4, 0x6604c4 },
+                { 0x04c8, 0x6604c8 },
+                { 0x04d0, 0x6604d0 },
+                { 0x04d4, 0x6604d4 },
+                { 0x04e0, 0x6604e0 },
+                { 0x04e4, 0x6604e4 },
+                { 0x04e8, 0x6604e8 },
+                { 0x04ec, 0x6604ec },
+                { 0x04f0, 0x6604f0 },
+                { 0x04f4, 0x6604f4 },
+                { 0x04f8, 0x6604f8 },
+                { 0x04fc, 0x6604fc },
+                { 0x0500, 0x660500 },
+                { 0x0504, 0x660504 },
+                { 0x0508, 0x660508 },
+                { 0x050c, 0x66050c },
+                { 0x0510, 0x660510 },
+                { 0x0514, 0x660514 },
+                { 0x0518, 0x660518 },
+                { 0x051c, 0x66051c },
+                { 0x0520, 0x660520 },
+                { 0x0524, 0x660524 },
+                { 0x052c, 0x66052c },
+                { 0x0530, 0x660530 },
+                { 0x054c, 0x66054c },
+                { 0x0550, 0x660550 },
+                { 0x0554, 0x660554 },
+                { 0x0558, 0x660558 },
+                { 0x055c, 0x66055c },
+                {}
+        }
+};
+
+const struct nv50_disp_mthd_chan
+nve0_disp_mast_mthd_chan = {
+        .name = "Core",
+        .addr = 0x000000,
+        .data = {
+                { "Global", 1, &nvd0_disp_mast_mthd_base },
+                {    "DAC", 3, &nvd0_disp_mast_mthd_dac  },
+                {    "SOR", 8, &nvd0_disp_mast_mthd_sor  },
+                {   "PIOR", 4, &nvd0_disp_mast_mthd_pior },
+                {   "HEAD", 4, &nve0_disp_mast_mthd_head },
+                {}
+        }
+};
+
+/*******************************************************************************
+ * EVO overlay channel objects
+ ******************************************************************************/
+
+static const struct nv50_disp_mthd_list
+nve0_disp_ovly_mthd_base = {
+        .mthd = 0x0000,
+        .data = {
+                { 0x0080, 0x665080 },
+                { 0x0084, 0x665084 },
+                { 0x0088, 0x665088 },
+                { 0x008c, 0x66508c },
+                { 0x0090, 0x665090 },
+                { 0x0094, 0x665094 },
+                { 0x00a0, 0x6650a0 },
+                { 0x00a4, 0x6650a4 },
+                { 0x00b0, 0x6650b0 },
+                { 0x00b4, 0x6650b4 },
+                { 0x00b8, 0x6650b8 },
+                { 0x00c0, 0x6650c0 },
+                { 0x00c4, 0x6650c4 },
+                { 0x00e0, 0x6650e0 },
+                { 0x00e4, 0x6650e4 },
+                { 0x00e8, 0x6650e8 },
+                { 0x0100, 0x665100 },
+                { 0x0104, 0x665104 },
+                { 0x0108, 0x665108 },
+                { 0x010c, 0x66510c },
+                { 0x0110, 0x665110 },
+                { 0x0118, 0x665118 },
+                { 0x011c, 0x66511c },
+                { 0x0120, 0x665120 },
+                { 0x0124, 0x665124 },
+                { 0x0130, 0x665130 },
+                { 0x0134, 0x665134 },
+                { 0x0138, 0x665138 },
+                { 0x013c, 0x66513c },
+                { 0x0140, 0x665140 },
+                { 0x0144, 0x665144 },
+                { 0x0148, 0x665148 },
+                { 0x014c, 0x66514c },
+                { 0x0150, 0x665150 },
+                { 0x0154, 0x665154 },
+                { 0x0158, 0x665158 },
+                { 0x015c, 0x66515c },
+                { 0x0160, 0x665160 },
+                { 0x0164, 0x665164 },
+                { 0x0168, 0x665168 },
+                { 0x016c, 0x66516c },
+                { 0x0400, 0x665400 },
+                { 0x0404, 0x665404 },
+                { 0x0408, 0x665408 },
+                { 0x040c, 0x66540c },
+                { 0x0410, 0x665410 },
+                {}
+        }
+};
+
+const struct nv50_disp_mthd_chan
+nve0_disp_ovly_mthd_chan = {
+        .name = "Overlay",
+        .addr = 0x001000,
+        .data = {
+                { "Global", 1, &nve0_disp_ovly_mthd_base },
+                {}
+        }
+};
+
+/*******************************************************************************
+ * Base display object
+ ******************************************************************************/
+
 static struct nouveau_oclass
 nve0_disp_sclass[] = {
         { NVE0_DISP_MAST_CLASS, &nvd0_disp_mast_ofuncs },
@@ -45,6 +214,10 @@ nve0_disp_base_oclass[] = {
         {}
 };
 
+/*******************************************************************************
+ * Display engine implementation
+ ******************************************************************************/
+
 static int
 nve0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                struct nouveau_oclass *oclass, void *data, u32 size,
@@ -77,13 +250,17 @@ nve0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-struct nouveau_oclass
-nve0_disp_oclass = {
-        .handle = NV_ENGINE(DISP, 0x91),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nve0_disp_oclass = &(struct nv50_disp_impl) {
+        .base.base.handle = NV_ENGINE(DISP, 0x91),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nve0_disp_ctor,
                 .dtor = _nouveau_disp_dtor,
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
-};
+        .mthd.core = &nve0_disp_mast_mthd_chan,
+        .mthd.base = &nvd0_disp_sync_mthd_chan,
+        .mthd.ovly = &nve0_disp_ovly_mthd_chan,
+        .mthd.prev = -0x020000,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c b/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c
index 05fee10e0c97..482585d375fa 100644
--- a/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/nvf0.c
@@ -29,6 +29,10 @@
 
 #include "nv50.h"
 
+/*******************************************************************************
+ * Base display object
+ ******************************************************************************/
+
 static struct nouveau_oclass
 nvf0_disp_sclass[] = {
         { NVF0_DISP_MAST_CLASS, &nvd0_disp_mast_ofuncs },
@@ -45,6 +49,10 @@ nvf0_disp_base_oclass[] = {
         {}
 };
 
+/*******************************************************************************
+ * Display engine implementation
+ ******************************************************************************/
+
 static int
 nvf0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                struct nouveau_oclass *oclass, void *data, u32 size,
@@ -77,13 +85,17 @@ nvf0_disp_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-struct nouveau_oclass
-nvf0_disp_oclass = {
-        .handle = NV_ENGINE(DISP, 0x92),
-        .ofuncs = &(struct nouveau_ofuncs) {
+struct nouveau_oclass *
+nvf0_disp_oclass = &(struct nv50_disp_impl) {
+        .base.base.handle = NV_ENGINE(DISP, 0x92),
+        .base.base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nvf0_disp_ctor,
                 .dtor = _nouveau_disp_dtor,
                 .init = _nouveau_disp_init,
                 .fini = _nouveau_disp_fini,
         },
-};
+        .mthd.core = &nve0_disp_mast_mthd_chan,
+        .mthd.base = &nvd0_disp_sync_mthd_chan,
+        .mthd.ovly = &nve0_disp_ovly_mthd_chan,
+        .mthd.prev = -0x020000,
+}.base.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/disp/priv.h b/drivers/gpu/drm/nouveau/core/engine/disp/priv.h
new file mode 100644
index 000000000000..cc3c7a4ca747
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/disp/priv.h
@@ -0,0 +1,10 @@
+#ifndef __NVKM_DISP_PRIV_H__
+#define __NVKM_DISP_PRIV_H__
+
+#include <engine/disp.h>
+
+struct nouveau_disp_impl {
+        struct nouveau_oclass base;
+};
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c
index 944e73ac485c..1cfb3bb90131 100644
--- a/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/dmaobj/nvd0.c
@@ -53,6 +53,9 @@ nvd0_dmaobj_bind(struct nouveau_dmaeng *dmaeng,
                 case NVF0_DISP_MAST_CLASS:
                 case NVF0_DISP_SYNC_CLASS:
                 case NVF0_DISP_OVLY_CLASS:
+                case GM107_DISP_MAST_CLASS:
+                case GM107_DISP_SYNC_CLASS:
+                case GM107_DISP_OVLY_CLASS:
                         break;
                 default:
                         return -EINVAL;
diff --git a/drivers/gpu/drm/nouveau/core/engine/falcon.c b/drivers/gpu/drm/nouveau/core/engine/falcon.c
index 5e077e4ed7f6..2914646c8709 100644
--- a/drivers/gpu/drm/nouveau/core/engine/falcon.c
+++ b/drivers/gpu/drm/nouveau/core/engine/falcon.c
@@ -119,7 +119,7 @@ _nouveau_falcon_init(struct nouveau_object *object)
                 snprintf(name, sizeof(name), "nouveau/nv%02x_fuc%03x",
                          device->chipset, falcon->addr >> 12);
 
-                ret = request_firmware(&fw, name, &device->pdev->dev);
+                ret = request_firmware(&fw, name, nv_device_base(device));
                 if (ret == 0) {
                         falcon->code.data = vmemdup(fw->data, fw->size);
                         falcon->code.size = fw->size;
@@ -138,7 +138,7 @@ _nouveau_falcon_init(struct nouveau_object *object)
                 snprintf(name, sizeof(name), "nouveau/nv%02x_fuc%03xd",
                          device->chipset, falcon->addr >> 12);
 
-                ret = request_firmware(&fw, name, &device->pdev->dev);
+                ret = request_firmware(&fw, name, nv_device_base(device));
                 if (ret) {
                         nv_error(falcon, "unable to load firmware data\n");
                         return ret;
@@ -153,7 +153,7 @@ _nouveau_falcon_init(struct nouveau_object *object)
                 snprintf(name, sizeof(name), "nouveau/nv%02x_fuc%03xc",
                          device->chipset, falcon->addr >> 12);
 
-                ret = request_firmware(&fw, name, &device->pdev->dev);
+                ret = request_firmware(&fw, name, nv_device_base(device));
                 if (ret) {
                         nv_error(falcon, "unable to load firmware code\n");
                         return ret;
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/base.c b/drivers/gpu/drm/nouveau/core/engine/fifo/base.c
index d3ec436d9cb5..6f9041ced9a2 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/base.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/base.c
@@ -86,7 +86,7 @@ nouveau_fifo_channel_create_(struct nouveau_object *parent,
         }
 
         /* map fifo control registers */
-        chan->user = ioremap(pci_resource_start(device->pdev, bar) + addr +
+        chan->user = ioremap(nv_device_resource_start(device, bar) + addr +
                              (chan->chid * size), size);
         if (!chan->user)
                 return -EFAULT;
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c
index b22a33f0702d..fa1e719872b7 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nvc0.c
@@ -41,8 +41,16 @@
 
 struct nvc0_fifo_priv {
         struct nouveau_fifo base;
-        struct nouveau_gpuobj *playlist[2];
-        int cur_playlist;
+
+        struct work_struct fault;
+        u64 mask;
+
+        struct {
+                struct nouveau_gpuobj *mem[2];
+                int active;
+                wait_queue_head_t wait;
+        } runlist;
+
         struct {
                 struct nouveau_gpuobj *mem;
                 struct nouveau_vma bar;
@@ -58,6 +66,11 @@ struct nvc0_fifo_base {
 
 struct nvc0_fifo_chan {
         struct nouveau_fifo_chan base;
+        enum {
+                STOPPED,
+                RUNNING,
+                KILLED
+        } state;
 };
 
 /*******************************************************************************
@@ -65,29 +78,33 @@ struct nvc0_fifo_chan {
  ******************************************************************************/
 
 static void
-nvc0_fifo_playlist_update(struct nvc0_fifo_priv *priv)
+nvc0_fifo_runlist_update(struct nvc0_fifo_priv *priv)
 {
         struct nouveau_bar *bar = nouveau_bar(priv);
         struct nouveau_gpuobj *cur;
         int i, p;
 
         mutex_lock(&nv_subdev(priv)->mutex);
-        cur = priv->playlist[priv->cur_playlist];
-        priv->cur_playlist = !priv->cur_playlist;
+        cur = priv->runlist.mem[priv->runlist.active];
+        priv->runlist.active = !priv->runlist.active;
 
         for (i = 0, p = 0; i < 128; i++) {
-                if (!(nv_rd32(priv, 0x003004 + (i * 8)) & 1))
-                        continue;
-                nv_wo32(cur, p + 0, i);
-                nv_wo32(cur, p + 4, 0x00000004);
-                p += 8;
+                struct nvc0_fifo_chan *chan = (void *)priv->base.channel[i];
+                if (chan && chan->state == RUNNING) {
+                        nv_wo32(cur, p + 0, i);
+                        nv_wo32(cur, p + 4, 0x00000004);
+                        p += 8;
+                }
         }
         bar->flush(bar);
 
         nv_wr32(priv, 0x002270, cur->addr >> 12);
         nv_wr32(priv, 0x002274, 0x01f00000 | (p >> 3));
-        if (!nv_wait(priv, 0x00227c, 0x00100000, 0x00000000))
-                nv_error(priv, "playlist update failed\n");
+
+        if (wait_event_timeout(priv->runlist.wait,
+                               !(nv_rd32(priv, 0x00227c) & 0x00100000),
+                               msecs_to_jiffies(2000)) == 0)
+                nv_error(priv, "runlist update timeout\n");
         mutex_unlock(&nv_subdev(priv)->mutex);
 }
 
@@ -239,30 +256,32 @@ nvc0_fifo_chan_init(struct nouveau_object *object)
                 return ret;
 
         nv_wr32(priv, 0x003000 + (chid * 8), 0xc0000000 | base->addr >> 12);
-        nv_wr32(priv, 0x003004 + (chid * 8), 0x001f0001);
-        nvc0_fifo_playlist_update(priv);
+
+        if (chan->state == STOPPED && (chan->state = RUNNING) == RUNNING) {
+                nv_wr32(priv, 0x003004 + (chid * 8), 0x001f0001);
+                nvc0_fifo_runlist_update(priv);
+        }
+
         return 0;
 }
 
+static void nvc0_fifo_intr_engine(struct nvc0_fifo_priv *priv);
+
 static int
 nvc0_fifo_chan_fini(struct nouveau_object *object, bool suspend)
 {
         struct nvc0_fifo_priv *priv = (void *)object->engine;
         struct nvc0_fifo_chan *chan = (void *)object;
         u32 chid = chan->base.chid;
-        u32 mask, engine;
 
-        nv_mask(priv, 0x003004 + (chid * 8), 0x00000001, 0x00000000);
-        nvc0_fifo_playlist_update(priv);
-        mask = nv_rd32(priv, 0x0025a4);
-        for (engine = 0; mask && engine < 16; engine++) {
-                if (!(mask & (1 << engine)))
-                        continue;
-                nv_mask(priv, 0x0025a8 + (engine * 4), 0x00000000, 0x00000000);
-                mask &= ~(1 << engine);
+        if (chan->state == RUNNING && (chan->state = STOPPED) == STOPPED) {
+                nv_mask(priv, 0x003004 + (chid * 8), 0x00000001, 0x00000000);
+                nvc0_fifo_runlist_update(priv);
         }
-        nv_wr32(priv, 0x003000 + (chid * 8), 0x00000000);
 
+        nvc0_fifo_intr_engine(priv);
+
+        nv_wr32(priv, 0x003000 + (chid * 8), 0x00000000);
         return nouveau_fifo_channel_fini(&chan->base, suspend);
 }
 
@@ -345,11 +364,177 @@ nvc0_fifo_cclass = {
  * PFIFO engine
  ******************************************************************************/
 
-static const struct nouveau_enum nvc0_fifo_fault_unit[] = {
+static inline int
+nvc0_fifo_engidx(struct nvc0_fifo_priv *priv, u32 engn)
+{
+        switch (engn) {
+        case NVDEV_ENGINE_GR   : engn = 0; break;
+        case NVDEV_ENGINE_BSP  : engn = 1; break;
+        case NVDEV_ENGINE_PPP  : engn = 2; break;
+        case NVDEV_ENGINE_VP   : engn = 3; break;
+        case NVDEV_ENGINE_COPY0: engn = 4; break;
+        case NVDEV_ENGINE_COPY1: engn = 5; break;
+        default:
+                return -1;
+        }
+
+        return engn;
+}
+
+static inline struct nouveau_engine *
+nvc0_fifo_engine(struct nvc0_fifo_priv *priv, u32 engn)
+{
+        switch (engn) {
+        case 0: engn = NVDEV_ENGINE_GR; break;
+        case 1: engn = NVDEV_ENGINE_BSP; break;
+        case 2: engn = NVDEV_ENGINE_PPP; break;
+        case 3: engn = NVDEV_ENGINE_VP; break;
+        case 4: engn = NVDEV_ENGINE_COPY0; break;
+        case 5: engn = NVDEV_ENGINE_COPY1; break;
+        default:
+                return NULL;
+        }
+
+        return nouveau_engine(priv, engn);
+}
+
+static void
+nvc0_fifo_recover_work(struct work_struct *work)
+{
+        struct nvc0_fifo_priv *priv = container_of(work, typeof(*priv), fault);
+        struct nouveau_object *engine;
+        unsigned long flags;
+        u32 engn, engm = 0;
+        u64 mask, todo;
+
+        spin_lock_irqsave(&priv->base.lock, flags);
+        mask = priv->mask;
+        priv->mask = 0ULL;
+        spin_unlock_irqrestore(&priv->base.lock, flags);
+
+        for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn))
+                engm |= 1 << nvc0_fifo_engidx(priv, engn);
+        nv_mask(priv, 0x002630, engm, engm);
+
+        for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) {
+                if ((engine = (void *)nouveau_engine(priv, engn))) {
+                        nv_ofuncs(engine)->fini(engine, false);
+                        WARN_ON(nv_ofuncs(engine)->init(engine));
+                }
+        }
+
+        nvc0_fifo_runlist_update(priv);
+        nv_wr32(priv, 0x00262c, engm);
+        nv_mask(priv, 0x002630, engm, 0x00000000);
+}
+
+static void
+nvc0_fifo_recover(struct nvc0_fifo_priv *priv, struct nouveau_engine *engine,
+                  struct nvc0_fifo_chan *chan)
+{
+        struct nouveau_object *engobj = nv_object(engine);
+        u32 chid = chan->base.chid;
+        unsigned long flags;
+
+        nv_error(priv, "%s engine fault on channel %d, recovering...\n",
+                       nv_subdev(engine)->name, chid);
+
+        nv_mask(priv, 0x003004 + (chid * 0x08), 0x00000001, 0x00000000);
+        chan->state = KILLED;
+
+        spin_lock_irqsave(&priv->base.lock, flags);
+        priv->mask |= 1ULL << nv_engidx(engobj);
+        spin_unlock_irqrestore(&priv->base.lock, flags);
+        schedule_work(&priv->fault);
+}
+
+static int
+nvc0_fifo_swmthd(struct nvc0_fifo_priv *priv, u32 chid, u32 mthd, u32 data)
+{
+        struct nvc0_fifo_chan *chan = NULL;
+        struct nouveau_handle *bind;
+        unsigned long flags;
+        int ret = -EINVAL;
+
+        spin_lock_irqsave(&priv->base.lock, flags);
+        if (likely(chid >= priv->base.min && chid <= priv->base.max))
+                chan = (void *)priv->base.channel[chid];
+        if (unlikely(!chan))
+                goto out;
+
+        bind = nouveau_namedb_get_class(nv_namedb(chan), 0x906e);
+        if (likely(bind)) {
+                if (!mthd || !nv_call(bind->object, mthd, data))
+                        ret = 0;
+                nouveau_namedb_put(bind);
+        }
+
+out:
+        spin_unlock_irqrestore(&priv->base.lock, flags);
+        return ret;
+}
+
+static const struct nouveau_enum
+nvc0_fifo_sched_reason[] = {
+        { 0x0a, "CTXSW_TIMEOUT" },
+        {}
+};
+
+static void
+nvc0_fifo_intr_sched_ctxsw(struct nvc0_fifo_priv *priv)
+{
+        struct nouveau_engine *engine;
+        struct nvc0_fifo_chan *chan;
+        u32 engn;
+
+        for (engn = 0; engn < 6; engn++) {
+                u32 stat = nv_rd32(priv, 0x002640 + (engn * 0x04));
+                u32 busy = (stat & 0x80000000);
+                u32 save = (stat & 0x00100000); /* maybe? */
+                u32 unk0 = (stat & 0x00040000);
+                u32 unk1 = (stat & 0x00001000);
+                u32 chid = (stat & 0x0000007f);
+                (void)save;
+
+                if (busy && unk0 && unk1) {
+                        if (!(chan = (void *)priv->base.channel[chid]))
+                                continue;
+                        if (!(engine = nvc0_fifo_engine(priv, engn)))
+                                continue;
+                        nvc0_fifo_recover(priv, engine, chan);
+                }
+        }
+}
+
+static void
+nvc0_fifo_intr_sched(struct nvc0_fifo_priv *priv)
+{
+        u32 intr = nv_rd32(priv, 0x00254c);
+        u32 code = intr & 0x000000ff;
+        const struct nouveau_enum *en;
+        char enunk[6] = "";
+
+        en = nouveau_enum_find(nvc0_fifo_sched_reason, code);
+        if (!en)
+                snprintf(enunk, sizeof(enunk), "UNK%02x", code);
+
+        nv_error(priv, "SCHED_ERROR [ %s ]\n", en ? en->name : enunk);
+
+        switch (code) {
+        case 0x0a:
+                nvc0_fifo_intr_sched_ctxsw(priv);
+                break;
+        default:
+                break;
+        }
+}
+
+static const struct nouveau_enum
+nvc0_fifo_fault_engine[] = {
         { 0x00, "PGRAPH", NULL, NVDEV_ENGINE_GR },
-        { 0x03, "PEEPHOLE" },
-        { 0x04, "BAR1" },
-        { 0x05, "BAR3" },
+        { 0x03, "PEEPHOLE", NULL, NVDEV_ENGINE_IFB },
+        { 0x04, "BAR1", NULL, NVDEV_SUBDEV_BAR },
+        { 0x05, "BAR3", NULL, NVDEV_SUBDEV_INSTMEM },
         { 0x07, "PFIFO", NULL, NVDEV_ENGINE_FIFO },
         { 0x10, "PBSP", NULL, NVDEV_ENGINE_BSP },
         { 0x11, "PPPP", NULL, NVDEV_ENGINE_PPP },
@@ -361,7 +546,8 @@ static const struct nouveau_enum nvc0_fifo_fault_unit[] = {
         {}
 };
 
-static const struct nouveau_enum nvc0_fifo_fault_reason[] = {
+static const struct nouveau_enum
+nvc0_fifo_fault_reason[] = {
         { 0x00, "PT_NOT_PRESENT" },
         { 0x01, "PT_TOO_SHORT" },
         { 0x02, "PAGE_NOT_PRESENT" },
@@ -374,7 +560,8 @@ static const struct nouveau_enum nvc0_fifo_fault_reason[] = {
         {}
 };
 
-static const struct nouveau_enum nvc0_fifo_fault_hubclient[] = {
+static const struct nouveau_enum
+nvc0_fifo_fault_hubclient[] = {
         { 0x01, "PCOPY0" },
         { 0x02, "PCOPY1" },
         { 0x04, "DISPATCH" },
@@ -392,7 +579,8 @@ static const struct nouveau_enum nvc0_fifo_fault_hubclient[] = {
         {}
 };
 
-static const struct nouveau_enum nvc0_fifo_fault_gpcclient[] = {
+static const struct nouveau_enum
+nvc0_fifo_fault_gpcclient[] = {
         { 0x01, "TEX" },
         { 0x0c, "ESETUP" },
         { 0x0e, "CTXCTL" },
@@ -400,92 +588,92 @@ static const struct nouveau_enum nvc0_fifo_fault_gpcclient[] = {
         {}
 };
 
-static const struct nouveau_bitfield nvc0_fifo_subfifo_intr[] = {
-/*      { 0x00008000, "" }      seen with null ib push */
-        { 0x00200000, "ILLEGAL_MTHD" },
-        { 0x00800000, "EMPTY_SUBC" },
-        {}
-};
-
 static void
-nvc0_fifo_isr_vm_fault(struct nvc0_fifo_priv *priv, int unit)
+nvc0_fifo_intr_fault(struct nvc0_fifo_priv *priv, int unit)
 {
         u32 inst = nv_rd32(priv, 0x002800 + (unit * 0x10));
         u32 valo = nv_rd32(priv, 0x002804 + (unit * 0x10));
         u32 vahi = nv_rd32(priv, 0x002808 + (unit * 0x10));
         u32 stat = nv_rd32(priv, 0x00280c + (unit * 0x10));
+        u32 gpc    = (stat & 0x1f000000) >> 24;
         u32 client = (stat & 0x00001f00) >> 8;
-        const struct nouveau_enum *en;
-        struct nouveau_engine *engine;
-        struct nouveau_object *engctx = NULL;
-
-        switch (unit) {
-        case 3: /* PEEPHOLE */
-                nv_mask(priv, 0x001718, 0x00000000, 0x00000000);
-                break;
-        case 4: /* BAR1 */
-                nv_mask(priv, 0x001704, 0x00000000, 0x00000000);
-                break;
-        case 5: /* BAR3 */
-                nv_mask(priv, 0x001714, 0x00000000, 0x00000000);
-                break;
-        default:
-                break;
+        u32 write  = (stat & 0x00000080);
+        u32 hub    = (stat & 0x00000040);
+        u32 reason = (stat & 0x0000000f);
+        struct nouveau_object *engctx = NULL, *object;
+        struct nouveau_engine *engine = NULL;
+        const struct nouveau_enum *er, *eu, *ec;
+        char erunk[6] = "";
+        char euunk[6] = "";
+        char ecunk[6] = "";
+        char gpcid[3] = "";
+
+        er = nouveau_enum_find(nvc0_fifo_fault_reason, reason);
+        if (!er)
+                snprintf(erunk, sizeof(erunk), "UNK%02X", reason);
+
+        eu = nouveau_enum_find(nvc0_fifo_fault_engine, unit);
+        if (eu) {
+                switch (eu->data2) {
+                case NVDEV_SUBDEV_BAR:
+                        nv_mask(priv, 0x001704, 0x00000000, 0x00000000);
+                        break;
+                case NVDEV_SUBDEV_INSTMEM:
+                        nv_mask(priv, 0x001714, 0x00000000, 0x00000000);
+                        break;
+                case NVDEV_ENGINE_IFB:
+                        nv_mask(priv, 0x001718, 0x00000000, 0x00000000);
+                        break;
+                default:
+                        engine = nouveau_engine(priv, eu->data2);
+                        if (engine)
+                                engctx = nouveau_engctx_get(engine, inst);
+                        break;
+                }
+        } else {
+                snprintf(euunk, sizeof(euunk), "UNK%02x", unit);
         }
 
-        nv_error(priv, "%s fault at 0x%010llx [", (stat & 0x00000080) ?
-                 "write" : "read", (u64)vahi << 32 | valo);
-        nouveau_enum_print(nvc0_fifo_fault_reason, stat & 0x0000000f);
-        pr_cont("] from ");
-        en = nouveau_enum_print(nvc0_fifo_fault_unit, unit);
-        if (stat & 0x00000040) {
-                pr_cont("/");
-                nouveau_enum_print(nvc0_fifo_fault_hubclient, client);
+        if (hub) {
+                ec = nouveau_enum_find(nvc0_fifo_fault_hubclient, client);
         } else {
-                pr_cont("/GPC%d/", (stat & 0x1f000000) >> 24);
-                nouveau_enum_print(nvc0_fifo_fault_gpcclient, client);
+                ec = nouveau_enum_find(nvc0_fifo_fault_gpcclient, client);
+                snprintf(gpcid, sizeof(gpcid), "%d", gpc);
         }
 
-        if (en && en->data2) {
-                engine = nouveau_engine(priv, en->data2);
-                if (engine)
-                        engctx = nouveau_engctx_get(engine, inst);
-
+        if (!ec)
+                snprintf(ecunk, sizeof(ecunk), "UNK%02x", client);
+
+        nv_error(priv, "%s fault at 0x%010llx [%s] from %s/%s%s%s%s on "
+                       "channel 0x%010llx [%s]\n", write ? "write" : "read",
+                 (u64)vahi << 32 | valo, er ? er->name : erunk,
+                 eu ? eu->name : euunk, hub ? "" : "GPC", gpcid, hub ? "" : "/",
+                 ec ? ec->name : ecunk, (u64)inst << 12,
+                 nouveau_client_name(engctx));
+
+        object = engctx;
+        while (object) {
+                switch (nv_mclass(object)) {
+                case NVC0_CHANNEL_IND_CLASS:
+                        nvc0_fifo_recover(priv, engine, (void *)object);
+                        break;
+                }
+                object = object->parent;
         }
-        pr_cont(" on channel 0x%010llx [%s]\n", (u64)inst << 12,
-                        nouveau_client_name(engctx));
 
         nouveau_engctx_put(engctx);
 }
 
-static int
-nvc0_fifo_swmthd(struct nvc0_fifo_priv *priv, u32 chid, u32 mthd, u32 data)
-{
-        struct nvc0_fifo_chan *chan = NULL;
-        struct nouveau_handle *bind;
-        unsigned long flags;
-        int ret = -EINVAL;
-
-        spin_lock_irqsave(&priv->base.lock, flags);
-        if (likely(chid >= priv->base.min && chid <= priv->base.max))
-                chan = (void *)priv->base.channel[chid];
-        if (unlikely(!chan))
-                goto out;
-
-        bind = nouveau_namedb_get_class(nv_namedb(chan), 0x906e);
-        if (likely(bind)) {
-                if (!mthd || !nv_call(bind->object, mthd, data))
-                        ret = 0;
-                nouveau_namedb_put(bind);
-        }
-
-out:
-        spin_unlock_irqrestore(&priv->base.lock, flags);
-        return ret;
-}
+static const struct nouveau_bitfield
+nvc0_fifo_pbdma_intr[] = {
+/*      { 0x00008000, "" }      seen with null ib push */
+        { 0x00200000, "ILLEGAL_MTHD" },
+        { 0x00800000, "EMPTY_SUBC" },
+        {}
+};
 
 static void
-nvc0_fifo_isr_subfifo_intr(struct nvc0_fifo_priv *priv, int unit)
+nvc0_fifo_intr_pbdma(struct nvc0_fifo_priv *priv, int unit)
 {
         u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000));
         u32 addr = nv_rd32(priv, 0x0400c0 + (unit * 0x2000));
@@ -501,11 +689,11 @@ nvc0_fifo_isr_subfifo_intr(struct nvc0_fifo_priv *priv, int unit)
         }
 
         if (show) {
-                nv_error(priv, "SUBFIFO%d:", unit);
-                nouveau_bitfield_print(nvc0_fifo_subfifo_intr, show);
+                nv_error(priv, "PBDMA%d:", unit);
+                nouveau_bitfield_print(nvc0_fifo_pbdma_intr, show);
                 pr_cont("\n");
                 nv_error(priv,
-                         "SUBFIFO%d: ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n",
+                         "PBDMA%d: ch %d [%s] subc %d mthd 0x%04x data 0x%08x\n",
                          unit, chid,
                          nouveau_client_name_for_fifo_chid(&priv->base, chid),
                          subc, mthd, data);
@@ -516,6 +704,56 @@ nvc0_fifo_isr_subfifo_intr(struct nvc0_fifo_priv *priv, int unit)
 }
 
 static void
+nvc0_fifo_intr_runlist(struct nvc0_fifo_priv *priv)
+{
+        u32 intr = nv_rd32(priv, 0x002a00);
+
+        if (intr & 0x10000000) {
+                wake_up(&priv->runlist.wait);
+                nv_wr32(priv, 0x002a00, 0x10000000);
+                intr &= ~0x10000000;
+        }
+
+        if (intr) {
+                nv_error(priv, "RUNLIST 0x%08x\n", intr);
+                nv_wr32(priv, 0x002a00, intr);
+        }
+}
+
+static void
+nvc0_fifo_intr_engine_unit(struct nvc0_fifo_priv *priv, int engn)
+{
+        u32 intr = nv_rd32(priv, 0x0025a8 + (engn * 0x04));
+        u32 inte = nv_rd32(priv, 0x002628);
+        u32 unkn;
+
+        for (unkn = 0; unkn < 8; unkn++) {
+                u32 ints = (intr >> (unkn * 0x04)) & inte;
+                if (ints & 0x1) {
+                        nouveau_event_trigger(priv->base.uevent, 0);
+                        ints &= ~1;
+                }
+                if (ints) {
+                        nv_error(priv, "ENGINE %d %d %01x", engn, unkn, ints);
+                        nv_mask(priv, 0x002628, ints, 0);
+                }
+        }
+
+        nv_wr32(priv, 0x0025a8 + (engn * 0x04), intr);
+}
+
+static void
+nvc0_fifo_intr_engine(struct nvc0_fifo_priv *priv)
+{
+        u32 mask = nv_rd32(priv, 0x0025a4);
+        while (mask) {
+                u32 unit = __ffs(mask);
+                nvc0_fifo_intr_engine_unit(priv, unit);
+                mask &= ~(1 << unit);
+        }
+}
+
+static void
 nvc0_fifo_intr(struct nouveau_subdev *subdev)
 {
         struct nvc0_fifo_priv *priv = (void *)subdev;
@@ -530,8 +768,7 @@ nvc0_fifo_intr(struct nouveau_subdev *subdev)
         }
 
         if (stat & 0x00000100) {
-                u32 intr = nv_rd32(priv, 0x00254c);
-                nv_warn(priv, "INTR 0x00000100: 0x%08x\n", intr);
+                nvc0_fifo_intr_sched(priv);
                 nv_wr32(priv, 0x002100, 0x00000100);
                 stat &= ~0x00000100;
         }
@@ -551,52 +788,41 @@ nvc0_fifo_intr(struct nouveau_subdev *subdev)
         }
 
         if (stat & 0x10000000) {
-                u32 units = nv_rd32(priv, 0x00259c);
-                u32 u = units;
-
-                while (u) {
-                        int i = ffs(u) - 1;
-                        nvc0_fifo_isr_vm_fault(priv, i);
-                        u &= ~(1 << i);
+                u32 mask = nv_rd32(priv, 0x00259c);
+                while (mask) {
+                        u32 unit = __ffs(mask);
+                        nvc0_fifo_intr_fault(priv, unit);
+                        nv_wr32(priv, 0x00259c, (1 << unit));
+                        mask &= ~(1 << unit);
                 }
-
-                nv_wr32(priv, 0x00259c, units);
                 stat &= ~0x10000000;
         }
 
         if (stat & 0x20000000) {
-                u32 units = nv_rd32(priv, 0x0025a0);
-                u32 u = units;
-
-                while (u) {
-                        int i = ffs(u) - 1;
-                        nvc0_fifo_isr_subfifo_intr(priv, i);
-                        u &= ~(1 << i);
+                u32 mask = nv_rd32(priv, 0x0025a0);
+                while (mask) {
+                        u32 unit = __ffs(mask);
+                        nvc0_fifo_intr_pbdma(priv, unit);
+                        nv_wr32(priv, 0x0025a0, (1 << unit));
+                        mask &= ~(1 << unit);
                 }
-
-                nv_wr32(priv, 0x0025a0, units);
                 stat &= ~0x20000000;
         }
 
         if (stat & 0x40000000) {
-                u32 intr0 = nv_rd32(priv, 0x0025a4);
-                u32 intr1 = nv_mask(priv, 0x002a00, 0x00000000, 0x00000);
-                nv_debug(priv, "INTR 0x40000000: 0x%08x 0x%08x\n",
-                               intr0, intr1);
+                nvc0_fifo_intr_runlist(priv);
                 stat &= ~0x40000000;
         }
 
         if (stat & 0x80000000) {
-                u32 intr = nv_mask(priv, 0x0025a8, 0x00000000, 0x00000000);
-                nouveau_event_trigger(priv->base.uevent, 0);
-                nv_debug(priv, "INTR 0x80000000: 0x%08x\n", intr);
+                nvc0_fifo_intr_engine(priv);
                 stat &= ~0x80000000;
         }
 
         if (stat) {
-                nv_fatal(priv, "unhandled status 0x%08x\n", stat);
+                nv_error(priv, "INTR 0x%08x\n", stat);
+                nv_mask(priv, 0x002140, stat, 0x00000000);
                 nv_wr32(priv, 0x002100, stat);
-                nv_wr32(priv, 0x002140, 0);
         }
 }
 
@@ -627,16 +853,20 @@ nvc0_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
+        INIT_WORK(&priv->fault, nvc0_fifo_recover_work);
+
         ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 0x1000, 0,
-                                &priv->playlist[0]);
+                                &priv->runlist.mem[0]);
         if (ret)
                 return ret;
 
         ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 0x1000, 0,
-                                &priv->playlist[1]);
+                                &priv->runlist.mem[1]);
         if (ret)
                 return ret;
 
+        init_waitqueue_head(&priv->runlist.wait);
+
         ret = nouveau_gpuobj_new(nv_object(priv), NULL, 128 * 0x1000, 0x1000, 0,
                                 &priv->user.mem);
         if (ret)
@@ -665,8 +895,8 @@ nvc0_fifo_dtor(struct nouveau_object *object)
 
         nouveau_gpuobj_unmap(&priv->user.bar);
         nouveau_gpuobj_ref(NULL, &priv->user.mem);
-        nouveau_gpuobj_ref(NULL, &priv->playlist[1]);
-        nouveau_gpuobj_ref(NULL, &priv->playlist[0]);
+        nouveau_gpuobj_ref(NULL, &priv->runlist.mem[0]);
+        nouveau_gpuobj_ref(NULL, &priv->runlist.mem[1]);
 
         nouveau_fifo_destroy(&priv->base);
 }
@@ -685,9 +915,9 @@ nvc0_fifo_init(struct nouveau_object *object)
         nv_wr32(priv, 0x002204, 0xffffffff);
 
         priv->spoon_nr = hweight32(nv_rd32(priv, 0x002204));
-        nv_debug(priv, "%d subfifo(s)\n", priv->spoon_nr);
+        nv_debug(priv, "%d PBDMA unit(s)\n", priv->spoon_nr);
 
-        /* assign engines to subfifos */
+        /* assign engines to PBDMAs */
         if (priv->spoon_nr >= 3) {
                 nv_wr32(priv, 0x002208, ~(1 << 0)); /* PGRAPH */
                 nv_wr32(priv, 0x00220c, ~(1 << 1)); /* PVP */
@@ -697,7 +927,7 @@ nvc0_fifo_init(struct nouveau_object *object)
                 nv_wr32(priv, 0x00221c, ~(1 << 1)); /* PCE1 */
         }
 
-        /* PSUBFIFO[n] */
+        /* PBDMA[n] */
         for (i = 0; i < priv->spoon_nr; i++) {
                 nv_mask(priv, 0x04013c + (i * 0x2000), 0x10000100, 0x00000000);
                 nv_wr32(priv, 0x040108 + (i * 0x2000), 0xffffffff); /* INTR */
@@ -707,10 +937,9 @@ nvc0_fifo_init(struct nouveau_object *object)
         nv_mask(priv, 0x002200, 0x00000001, 0x00000001);
         nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12);
 
-        nv_wr32(priv, 0x002a00, 0xffffffff); /* clears PFIFO.INTR bit 30 */
         nv_wr32(priv, 0x002100, 0xffffffff);
-        nv_wr32(priv, 0x002140, 0x3fffffff);
-        nv_wr32(priv, 0x002628, 0x00000001); /* makes mthd 0x20 work */
+        nv_wr32(priv, 0x002140, 0x7fffffff);
+        nv_wr32(priv, 0x002628, 0x00000001); /* ENGINE_INTR_EN */
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c b/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
index 54c1b5b471cd..a9a1a9c9f9f2 100644
--- a/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/fifo/nve0.c
@@ -60,10 +60,15 @@ static const struct {
 struct nve0_fifo_engn {
         struct nouveau_gpuobj *runlist[2];
         int cur_runlist;
+        wait_queue_head_t wait;
 };
 
 struct nve0_fifo_priv {
         struct nouveau_fifo base;
+
+        struct work_struct fault;
+        u64 mask;
+
         struct nve0_fifo_engn engine[FIFO_ENGINE_NR];
         struct {
                 struct nouveau_gpuobj *mem;
@@ -81,6 +86,11 @@ struct nve0_fifo_base {
 struct nve0_fifo_chan {
         struct nouveau_fifo_chan base;
         u32 engine;
+        enum {
+                STOPPED,
+                RUNNING,
+                KILLED
+        } state;
 };
 
 /*******************************************************************************
@@ -93,7 +103,6 @@ nve0_fifo_runlist_update(struct nve0_fifo_priv *priv, u32 engine)
         struct nouveau_bar *bar = nouveau_bar(priv);
         struct nve0_fifo_engn *engn = &priv->engine[engine];
         struct nouveau_gpuobj *cur;
-        u32 match = (engine << 16) | 0x00000001;
         int i, p;
 
         mutex_lock(&nv_subdev(priv)->mutex);
@@ -101,18 +110,21 @@ nve0_fifo_runlist_update(struct nve0_fifo_priv *priv, u32 engine)
         engn->cur_runlist = !engn->cur_runlist;
 
         for (i = 0, p = 0; i < priv->base.max; i++) {
-                u32 ctrl = nv_rd32(priv, 0x800004 + (i * 8)) & 0x001f0001;
-                if (ctrl != match)
-                        continue;
-                nv_wo32(cur, p + 0, i);
-                nv_wo32(cur, p + 4, 0x00000000);
-                p += 8;
+                struct nve0_fifo_chan *chan = (void *)priv->base.channel[i];
+                if (chan && chan->state == RUNNING && chan->engine == engine) {
+                        nv_wo32(cur, p + 0, i);
+                        nv_wo32(cur, p + 4, 0x00000000);
+                        p += 8;
+                }
         }
         bar->flush(bar);
 
         nv_wr32(priv, 0x002270, cur->addr >> 12);
         nv_wr32(priv, 0x002274, (engine << 20) | (p >> 3));
-        if (!nv_wait(priv, 0x002284 + (engine * 8), 0x00100000, 0x00000000))
+
+        if (wait_event_timeout(engn->wait, !(nv_rd32(priv, 0x002284 +
+                               (engine * 0x08)) & 0x00100000),
+                                msecs_to_jiffies(2000)) == 0)
                 nv_error(priv, "runlist %d update timeout\n", engine);
         mutex_unlock(&nv_subdev(priv)->mutex);
 }
@@ -129,9 +141,11 @@ nve0_fifo_context_attach(struct nouveau_object *parent,
 
         switch (nv_engidx(object->engine)) {
         case NVDEV_ENGINE_SW   :
+                return 0;
         case NVDEV_ENGINE_COPY0:
         case NVDEV_ENGINE_COPY1:
         case NVDEV_ENGINE_COPY2:
+                nv_engctx(ectx)->addr = nv_gpuobj(base)->addr >> 12;
                 return 0;
         case NVDEV_ENGINE_GR   : addr = 0x0210; break;
         case NVDEV_ENGINE_BSP  : addr = 0x0270; break;
@@ -279,9 +293,13 @@ nve0_fifo_chan_init(struct nouveau_object *object)
 
         nv_mask(priv, 0x800004 + (chid * 8), 0x000f0000, chan->engine << 16);
         nv_wr32(priv, 0x800000 + (chid * 8), 0x80000000 | base->addr >> 12);
-        nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400);
-        nve0_fifo_runlist_update(priv, chan->engine);
-        nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400);
+
+        if (chan->state == STOPPED && (chan->state = RUNNING) == RUNNING) {
+                nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400);
+                nve0_fifo_runlist_update(priv, chan->engine);
+                nv_mask(priv, 0x800004 + (chid * 8), 0x00000400, 0x00000400);
+        }
+
         return 0;
 }
 
@@ -292,10 +310,12 @@ nve0_fifo_chan_fini(struct nouveau_object *object, bool suspend)
         struct nve0_fifo_chan *chan = (void *)object;
         u32 chid = chan->base.chid;
 
-        nv_mask(priv, 0x800004 + (chid * 8), 0x00000800, 0x00000800);
-        nve0_fifo_runlist_update(priv, chan->engine);
-        nv_wr32(priv, 0x800000 + (chid * 8), 0x00000000);
+        if (chan->state == RUNNING && (chan->state = STOPPED) == STOPPED) {
+                nv_mask(priv, 0x800004 + (chid * 8), 0x00000800, 0x00000800);
+                nve0_fifo_runlist_update(priv, chan->engine);
+        }
 
+        nv_wr32(priv, 0x800000 + (chid * 8), 0x00000000);
         return nouveau_fifo_channel_fini(&chan->base, suspend);
 }
 
@@ -377,14 +397,211 @@ nve0_fifo_cclass = {
  * PFIFO engine
  ******************************************************************************/
 
-static const struct nouveau_enum nve0_fifo_sched_reason[] = {
+static inline int
+nve0_fifo_engidx(struct nve0_fifo_priv *priv, u32 engn)
+{
+        switch (engn) {
+        case NVDEV_ENGINE_GR   :
+        case NVDEV_ENGINE_COPY2: engn = 0; break;
+        case NVDEV_ENGINE_BSP  : engn = 1; break;
+        case NVDEV_ENGINE_PPP  : engn = 2; break;
+        case NVDEV_ENGINE_VP   : engn = 3; break;
+        case NVDEV_ENGINE_COPY0: engn = 4; break;
+        case NVDEV_ENGINE_COPY1: engn = 5; break;
+        case NVDEV_ENGINE_VENC : engn = 6; break;
+        default:
+                return -1;
+        }
+
+        return engn;
+}
+
+static inline struct nouveau_engine *
+nve0_fifo_engine(struct nve0_fifo_priv *priv, u32 engn)
+{
+        if (engn >= ARRAY_SIZE(fifo_engine))
+                return NULL;
+        return nouveau_engine(priv, fifo_engine[engn].subdev);
+}
+
+static void
+nve0_fifo_recover_work(struct work_struct *work)
+{
+        struct nve0_fifo_priv *priv = container_of(work, typeof(*priv), fault);
+        struct nouveau_object *engine;
+        unsigned long flags;
+        u32 engn, engm = 0;
+        u64 mask, todo;
+
+        spin_lock_irqsave(&priv->base.lock, flags);
+        mask = priv->mask;
+        priv->mask = 0ULL;
+        spin_unlock_irqrestore(&priv->base.lock, flags);
+
+        for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn))
+                engm |= 1 << nve0_fifo_engidx(priv, engn);
+        nv_mask(priv, 0x002630, engm, engm);
+
+        for (todo = mask; engn = __ffs64(todo), todo; todo &= ~(1 << engn)) {
+                if ((engine = (void *)nouveau_engine(priv, engn))) {
+                        nv_ofuncs(engine)->fini(engine, false);
+                        WARN_ON(nv_ofuncs(engine)->init(engine));
+                }
+                nve0_fifo_runlist_update(priv, nve0_fifo_engidx(priv, engn));
+        }
+
+        nv_wr32(priv, 0x00262c, engm);
+        nv_mask(priv, 0x002630, engm, 0x00000000);
+}
+
+static void
+nve0_fifo_recover(struct nve0_fifo_priv *priv, struct nouveau_engine *engine,
+                  struct nve0_fifo_chan *chan)
+{
+        struct nouveau_object *engobj = nv_object(engine);
+        u32 chid = chan->base.chid;
+        unsigned long flags;
+
+        nv_error(priv, "%s engine fault on channel %d, recovering...\n",
+                       nv_subdev(engine)->name, chid);
+
+        nv_mask(priv, 0x800004 + (chid * 0x08), 0x00000800, 0x00000800);
+        chan->state = KILLED;
+
+        spin_lock_irqsave(&priv->base.lock, flags);
+        priv->mask |= 1ULL << nv_engidx(engobj);
+        spin_unlock_irqrestore(&priv->base.lock, flags);
+        schedule_work(&priv->fault);
+}
+
+static int
+nve0_fifo_swmthd(struct nve0_fifo_priv *priv, u32 chid, u32 mthd, u32 data)
+{
+        struct nve0_fifo_chan *chan = NULL;
+        struct nouveau_handle *bind;
+        unsigned long flags;
+        int ret = -EINVAL;
+
+        spin_lock_irqsave(&priv->base.lock, flags);
+        if (likely(chid >= priv->base.min && chid <= priv->base.max))
+                chan = (void *)priv->base.channel[chid];
+        if (unlikely(!chan))
+                goto out;
+
+        bind = nouveau_namedb_get_class(nv_namedb(chan), 0x906e);
+        if (likely(bind)) {
+                if (!mthd || !nv_call(bind->object, mthd, data))
+                        ret = 0;
+                nouveau_namedb_put(bind);
+        }
+
+out:
+        spin_unlock_irqrestore(&priv->base.lock, flags);
+        return ret;
+}
+
+static const struct nouveau_enum
+nve0_fifo_bind_reason[] = {
+        { 0x01, "BIND_NOT_UNBOUND" },
+        { 0x02, "SNOOP_WITHOUT_BAR1" },
+        { 0x03, "UNBIND_WHILE_RUNNING" },
+        { 0x05, "INVALID_RUNLIST" },
+        { 0x06, "INVALID_CTX_TGT" },
+        { 0x0b, "UNBIND_WHILE_PARKED" },
+        {}
+};
+
+static void
+nve0_fifo_intr_bind(struct nve0_fifo_priv *priv)
+{
+        u32 intr = nv_rd32(priv, 0x00252c);
+        u32 code = intr & 0x000000ff;
+        const struct nouveau_enum *en;
+        char enunk[6] = "";
+
+        en = nouveau_enum_find(nve0_fifo_bind_reason, code);
+        if (!en)
+                snprintf(enunk, sizeof(enunk), "UNK%02x", code);
+
+        nv_error(priv, "BIND_ERROR [ %s ]\n", en ? en->name : enunk);
+}
+
+static const struct nouveau_enum
+nve0_fifo_sched_reason[] = {
         { 0x0a, "CTXSW_TIMEOUT" },
         {}
 };
 
-static const struct nouveau_enum nve0_fifo_fault_engine[] = {
+static void
+nve0_fifo_intr_sched_ctxsw(struct nve0_fifo_priv *priv)
+{
+        struct nouveau_engine *engine;
+        struct nve0_fifo_chan *chan;
+        u32 engn;
+
+        for (engn = 0; engn < ARRAY_SIZE(fifo_engine); engn++) {
+                u32 stat = nv_rd32(priv, 0x002640 + (engn * 0x04));
+                u32 busy = (stat & 0x80000000);
+                u32 next = (stat & 0x07ff0000) >> 16;
+                u32 chsw = (stat & 0x00008000);
+                u32 save = (stat & 0x00004000);
+                u32 load = (stat & 0x00002000);
+                u32 prev = (stat & 0x000007ff);
+                u32 chid = load ? next : prev;
+                (void)save;
+
+                if (busy && chsw) {
+                        if (!(chan = (void *)priv->base.channel[chid]))
+                                continue;
+                        if (!(engine = nve0_fifo_engine(priv, engn)))
+                                continue;
+                        nve0_fifo_recover(priv, engine, chan);
+                }
+        }
+}
+
+static void
+nve0_fifo_intr_sched(struct nve0_fifo_priv *priv)
+{
+        u32 intr = nv_rd32(priv, 0x00254c);
+        u32 code = intr & 0x000000ff;
+        const struct nouveau_enum *en;
+        char enunk[6] = "";
+
+        en = nouveau_enum_find(nve0_fifo_sched_reason, code);
+        if (!en)
+                snprintf(enunk, sizeof(enunk), "UNK%02x", code);
+
+        nv_error(priv, "SCHED_ERROR [ %s ]\n", en ? en->name : enunk);
+
+        switch (code) {
+        case 0x0a:
+                nve0_fifo_intr_sched_ctxsw(priv);
+                break;
+        default:
+                break;
+        }
+}
+
+static void
+nve0_fifo_intr_chsw(struct nve0_fifo_priv *priv)
+{
+        u32 stat = nv_rd32(priv, 0x00256c);
+        nv_error(priv, "CHSW_ERROR 0x%08x\n", stat);
+        nv_wr32(priv, 0x00256c, stat);
+}
+
+static void
+nve0_fifo_intr_dropped_fault(struct nve0_fifo_priv *priv)
+{
+        u32 stat = nv_rd32(priv, 0x00259c);
+        nv_error(priv, "DROPPED_MMU_FAULT 0x%08x\n", stat);
+}
+
+static const struct nouveau_enum
+nve0_fifo_fault_engine[] = {
         { 0x00, "GR", NULL, NVDEV_ENGINE_GR },
-        { 0x03, "IFB" },
+        { 0x03, "IFB", NULL, NVDEV_ENGINE_IFB },
         { 0x04, "BAR1", NULL, NVDEV_SUBDEV_BAR },
         { 0x05, "BAR3", NULL, NVDEV_SUBDEV_INSTMEM },
         { 0x07, "PBDMA0", NULL, NVDEV_ENGINE_FIFO },
@@ -402,7 +619,8 @@ static const struct nouveau_enum nve0_fifo_fault_engine[] = {
         {}
 };
 
-static const struct nouveau_enum nve0_fifo_fault_reason[] = {
+static const struct nouveau_enum
+nve0_fifo_fault_reason[] = {
         { 0x00, "PDE" },
         { 0x01, "PDE_SIZE" },
         { 0x02, "PTE" },
@@ -422,7 +640,8 @@ static const struct nouveau_enum nve0_fifo_fault_reason[] = {
         {}
 };
 
-static const struct nouveau_enum nve0_fifo_fault_hubclient[] = {
+static const struct nouveau_enum
+nve0_fifo_fault_hubclient[] = {
         { 0x00, "VIP" },
         { 0x01, "CE0" },
         { 0x02, "CE1" },
@@ -458,7 +677,8 @@ static const struct nouveau_enum nve0_fifo_fault_hubclient[] = {
         {}
 };
 
-static const struct nouveau_enum nve0_fifo_fault_gpcclient[] = {
+static const struct nouveau_enum
+nve0_fifo_fault_gpcclient[] = {
         { 0x00, "L1_0" }, { 0x01, "T1_0" }, { 0x02, "PE_0" },
         { 0x03, "L1_1" }, { 0x04, "T1_1" }, { 0x05, "PE_1" },
         { 0x06, "L1_2" }, { 0x07, "T1_2" }, { 0x08, "PE_2" },
@@ -483,6 +703,82 @@ static const struct nouveau_enum nve0_fifo_fault_gpcclient[] = {
         {}
 };
 
+static void
+nve0_fifo_intr_fault(struct nve0_fifo_priv *priv, int unit)
+{
+        u32 inst = nv_rd32(priv, 0x002800 + (unit * 0x10));
+        u32 valo = nv_rd32(priv, 0x002804 + (unit * 0x10));
+        u32 vahi = nv_rd32(priv, 0x002808 + (unit * 0x10));
+        u32 stat = nv_rd32(priv, 0x00280c + (unit * 0x10));
+        u32 gpc    = (stat & 0x1f000000) >> 24;
+        u32 client = (stat & 0x00001f00) >> 8;
+        u32 write  = (stat & 0x00000080);
+        u32 hub    = (stat & 0x00000040);
+        u32 reason = (stat & 0x0000000f);
+        struct nouveau_object *engctx = NULL, *object;
+        struct nouveau_engine *engine = NULL;
+        const struct nouveau_enum *er, *eu, *ec;
+        char erunk[6] = "";
+        char euunk[6] = "";
+        char ecunk[6] = "";
+        char gpcid[3] = "";
+
+        er = nouveau_enum_find(nve0_fifo_fault_reason, reason);
+        if (!er)
+                snprintf(erunk, sizeof(erunk), "UNK%02X", reason);
+
+        eu = nouveau_enum_find(nve0_fifo_fault_engine, unit);
+        if (eu) {
+                switch (eu->data2) {
+                case NVDEV_SUBDEV_BAR:
+                        nv_mask(priv, 0x001704, 0x00000000, 0x00000000);
+                        break;
+                case NVDEV_SUBDEV_INSTMEM:
+                        nv_mask(priv, 0x001714, 0x00000000, 0x00000000);
+                        break;
+                case NVDEV_ENGINE_IFB:
+                        nv_mask(priv, 0x001718, 0x00000000, 0x00000000);
+                        break;
+                default:
+                        engine = nouveau_engine(priv, eu->data2);
+                        if (engine)
+                                engctx = nouveau_engctx_get(engine, inst);
+                        break;
+                }
+        } else {
+                snprintf(euunk, sizeof(euunk), "UNK%02x", unit);
+        }
+
+        if (hub) {
+                ec = nouveau_enum_find(nve0_fifo_fault_hubclient, client);
+        } else {
+                ec = nouveau_enum_find(nve0_fifo_fault_gpcclient, client);
+                snprintf(gpcid, sizeof(gpcid), "%d", gpc);
+        }
+
+        if (!ec)
+                snprintf(ecunk, sizeof(ecunk), "UNK%02x", client);
+
+        nv_error(priv, "%s fault at 0x%010llx [%s] from %s/%s%s%s%s on "
+                       "channel 0x%010llx [%s]\n", write ? "write" : "read",
+                 (u64)vahi << 32 | valo, er ? er->name : erunk,
+                 eu ? eu->name : euunk, hub ? "" : "GPC", gpcid, hub ? "" : "/",
+                 ec ? ec->name : ecunk, (u64)inst << 12,
+                 nouveau_client_name(engctx));
+
+        object = engctx;
+        while (object) {
+                switch (nv_mclass(object)) {
+                case NVE0_CHANNEL_IND_CLASS:
+                        nve0_fifo_recover(priv, engine, (void *)object);
+                        break;
+                }
+                object = object->parent;
+        }
+
+        nouveau_engctx_put(engctx);
+}
+
 static const struct nouveau_bitfield nve0_fifo_pbdma_intr[] = {
         { 0x00000001, "MEMREQ" },
         { 0x00000002, "MEMACK_TIMEOUT" },
@@ -518,104 +814,6 @@ static const struct nouveau_bitfield nve0_fifo_pbdma_intr[] = {
 };
 
 static void
-nve0_fifo_intr_sched(struct nve0_fifo_priv *priv)
-{
-        u32 intr = nv_rd32(priv, 0x00254c);
-        u32 code = intr & 0x000000ff;
-        nv_error(priv, "SCHED_ERROR [");
-        nouveau_enum_print(nve0_fifo_sched_reason, code);
-        pr_cont("]\n");
-}
-
-static void
-nve0_fifo_intr_chsw(struct nve0_fifo_priv *priv)
-{
-        u32 stat = nv_rd32(priv, 0x00256c);
-        nv_error(priv, "CHSW_ERROR 0x%08x\n", stat);
-        nv_wr32(priv, 0x00256c, stat);
-}
-
-static void
-nve0_fifo_intr_dropped_fault(struct nve0_fifo_priv *priv)
-{
-        u32 stat = nv_rd32(priv, 0x00259c);
-        nv_error(priv, "DROPPED_MMU_FAULT 0x%08x\n", stat);
-}
-
-static void
-nve0_fifo_intr_fault(struct nve0_fifo_priv *priv, int unit)
-{
-        u32 inst = nv_rd32(priv, 0x2800 + (unit * 0x10));
-        u32 valo = nv_rd32(priv, 0x2804 + (unit * 0x10));
-        u32 vahi = nv_rd32(priv, 0x2808 + (unit * 0x10));
-        u32 stat = nv_rd32(priv, 0x280c + (unit * 0x10));
-        u32 client = (stat & 0x00001f00) >> 8;
-        struct nouveau_engine *engine = NULL;
-        struct nouveau_object *engctx = NULL;
-        const struct nouveau_enum *en;
-        const char *name = "unknown";
-
-        nv_error(priv, "PFIFO: %s fault at 0x%010llx [", (stat & 0x00000080) ?
-                       "write" : "read", (u64)vahi << 32 | valo);
-        nouveau_enum_print(nve0_fifo_fault_reason, stat & 0x0000000f);
-        pr_cont("] from ");
-        en = nouveau_enum_print(nve0_fifo_fault_engine, unit);
-        if (stat & 0x00000040) {
-                pr_cont("/");
-                nouveau_enum_print(nve0_fifo_fault_hubclient, client);
-        } else {
-                pr_cont("/GPC%d/", (stat & 0x1f000000) >> 24);
-                nouveau_enum_print(nve0_fifo_fault_gpcclient, client);
-        }
-
-        if (en && en->data2) {
-                if (en->data2 == NVDEV_SUBDEV_BAR) {
-                        nv_mask(priv, 0x001704, 0x00000000, 0x00000000);
-                        name = "BAR1";
-                } else
-                if (en->data2 == NVDEV_SUBDEV_INSTMEM) {
-                        nv_mask(priv, 0x001714, 0x00000000, 0x00000000);
-                        name = "BAR3";
-                } else {
-                        engine = nouveau_engine(priv, en->data2);
-                        if (engine) {
-                                engctx = nouveau_engctx_get(engine, inst);
-                                name   = nouveau_client_name(engctx);
-                        }
-                }
-        }
-        pr_cont(" on channel 0x%010llx [%s]\n", (u64)inst << 12, name);
-
-        nouveau_engctx_put(engctx);
-}
-
-static int
-nve0_fifo_swmthd(struct nve0_fifo_priv *priv, u32 chid, u32 mthd, u32 data)
-{
-        struct nve0_fifo_chan *chan = NULL;
-        struct nouveau_handle *bind;
-        unsigned long flags;
-        int ret = -EINVAL;
-
-        spin_lock_irqsave(&priv->base.lock, flags);
-        if (likely(chid >= priv->base.min && chid <= priv->base.max))
-                chan = (void *)priv->base.channel[chid];
-        if (unlikely(!chan))
-                goto out;
-
-        bind = nouveau_namedb_get_class(nv_namedb(chan), 0x906e);
-        if (likely(bind)) {
-                if (!mthd || !nv_call(bind->object, mthd, data))
-                        ret = 0;
-                nouveau_namedb_put(bind);
-        }
-
-out:
-        spin_unlock_irqrestore(&priv->base.lock, flags);
-        return ret;
-}
-
-static void
 nve0_fifo_intr_pbdma(struct nve0_fifo_priv *priv, int unit)
 {
         u32 stat = nv_rd32(priv, 0x040108 + (unit * 0x2000));
@@ -647,6 +845,24 @@ nve0_fifo_intr_pbdma(struct nve0_fifo_priv *priv, int unit)
 }
 
 static void
+nve0_fifo_intr_runlist(struct nve0_fifo_priv *priv)
+{
+        u32 mask = nv_rd32(priv, 0x002a00);
+        while (mask) {
+                u32 engn = __ffs(mask);
+                wake_up(&priv->engine[engn].wait);
+                nv_wr32(priv, 0x002a00, 1 << engn);
+                mask &= ~(1 << engn);
+        }
+}
+
+static void
+nve0_fifo_intr_engine(struct nve0_fifo_priv *priv)
+{
+        nouveau_event_trigger(priv->base.uevent, 0);
+}
+
+static void
 nve0_fifo_intr(struct nouveau_subdev *subdev)
 {
         struct nve0_fifo_priv *priv = (void *)subdev;
@@ -654,8 +870,7 @@ nve0_fifo_intr(struct nouveau_subdev *subdev)
         u32 stat = nv_rd32(priv, 0x002100) & mask;
 
         if (stat & 0x00000001) {
-                u32 stat = nv_rd32(priv, 0x00252c);
-                nv_error(priv, "BIND_ERROR 0x%08x\n", stat);
+                nve0_fifo_intr_bind(priv);
                 nv_wr32(priv, 0x002100, 0x00000001);
                 stat &= ~0x00000001;
         }
@@ -697,55 +912,42 @@ nve0_fifo_intr(struct nouveau_subdev *subdev)
         }
 
         if (stat & 0x10000000) {
-                u32 units = nv_rd32(priv, 0x00259c);
-                u32 u = units;
-
-                while (u) {
-                        int i = ffs(u) - 1;
-                        nve0_fifo_intr_fault(priv, i);
-                        u &= ~(1 << i);
+                u32 mask = nv_rd32(priv, 0x00259c);
+                while (mask) {
+                        u32 unit = __ffs(mask);
+                        nve0_fifo_intr_fault(priv, unit);
+                        nv_wr32(priv, 0x00259c, (1 << unit));
+                        mask &= ~(1 << unit);
                 }
-
-                nv_wr32(priv, 0x00259c, units);
                 stat &= ~0x10000000;
         }
 
         if (stat & 0x20000000) {
                 u32 mask = nv_rd32(priv, 0x0025a0);
-                u32 temp = mask;
-
-                while (temp) {
-                        u32 unit = ffs(temp) - 1;
+                while (mask) {
+                        u32 unit = __ffs(mask);
                         nve0_fifo_intr_pbdma(priv, unit);
-                        temp &= ~(1 << unit);
+                        nv_wr32(priv, 0x0025a0, (1 << unit));
+                        mask &= ~(1 << unit);
                 }
-
-                nv_wr32(priv, 0x0025a0, mask);
                 stat &= ~0x20000000;
         }
 
         if (stat & 0x40000000) {
-                u32 mask = nv_mask(priv, 0x002a00, 0x00000000, 0x00000000);
-
-                while (mask) {
-                        u32 engn = ffs(mask) - 1;
-                        /* runlist event, not currently used */
-                        mask &= ~(1 << engn);
-                }
-
+                nve0_fifo_intr_runlist(priv);
                 stat &= ~0x40000000;
         }
 
         if (stat & 0x80000000) {
-                nouveau_event_trigger(priv->base.uevent, 0);
+                nve0_fifo_intr_engine(priv);
                 nv_wr32(priv, 0x002100, 0x80000000);
                 stat &= ~0x80000000;
         }
 
         if (stat) {
-                nv_fatal(priv, "unhandled status 0x%08x\n", stat);
+                nv_error(priv, "INTR 0x%08x\n", stat);
+                nv_mask(priv, 0x002140, stat, 0x00000000);
                 nv_wr32(priv, 0x002100, stat);
-                nv_wr32(priv, 0x002140, 0);
         }
 }
 
@@ -802,9 +1004,8 @@ nve0_fifo_init(struct nouveau_object *object)
 
         nv_wr32(priv, 0x002254, 0x10000000 | priv->user.bar.offset >> 12);
 
-        nv_wr32(priv, 0x002a00, 0xffffffff);
         nv_wr32(priv, 0x002100, 0xffffffff);
-        nv_wr32(priv, 0x002140, 0x3fffffff);
+        nv_wr32(priv, 0x002140, 0x7fffffff);
         return 0;
 }
 
@@ -840,6 +1041,8 @@ nve0_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
+        INIT_WORK(&priv->fault, nve0_fifo_recover_work);
+
         for (i = 0; i < FIFO_ENGINE_NR; i++) {
                 ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x8000, 0x1000,
                                          0, &priv->engine[i].runlist[0]);
@@ -850,10 +1053,12 @@ nve0_fifo_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                                          0, &priv->engine[i].runlist[1]);
                 if (ret)
                         return ret;
+
+                init_waitqueue_head(&priv->engine[i].wait);
         }
 
-        ret = nouveau_gpuobj_new(nv_object(priv), NULL, 4096 * 0x200, 0x1000,
-                                 NVOBJ_FLAG_ZERO_ALLOC, &priv->user.mem);
+        ret = nouveau_gpuobj_new(nv_object(priv), NULL, impl->channels * 0x200,
+                                0x1000, NVOBJ_FLAG_ZERO_ALLOC, &priv->user.mem);
         if (ret)
                 return ret;
 
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxgm107.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxgm107.c
new file mode 100644
index 000000000000..1dc37b1ddbfa
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxgm107.c
@@ -0,0 +1,989 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include "ctxnvc0.h"
+
+/*******************************************************************************
+ * PGRAPH context register lists
+ ******************************************************************************/
+
+static const struct nvc0_graph_init
+gm107_grctx_init_icmd_0[] = {
+        { 0x001000,   1, 0x01, 0x00000004 },
+        { 0x000039,   3, 0x01, 0x00000000 },
+        { 0x0000a9,   1, 0x01, 0x0000ffff },
+        { 0x000038,   1, 0x01, 0x0fac6881 },
+        { 0x00003d,   1, 0x01, 0x00000001 },
+        { 0x0000e8,   8, 0x01, 0x00000400 },
+        { 0x000078,   8, 0x01, 0x00000300 },
+        { 0x000050,   1, 0x01, 0x00000011 },
+        { 0x000058,   8, 0x01, 0x00000008 },
+        { 0x000208,   8, 0x01, 0x00000001 },
+        { 0x000081,   1, 0x01, 0x00000001 },
+        { 0x000085,   1, 0x01, 0x00000004 },
+        { 0x000088,   1, 0x01, 0x00000400 },
+        { 0x000090,   1, 0x01, 0x00000300 },
+        { 0x000098,   1, 0x01, 0x00001001 },
+        { 0x0000e3,   1, 0x01, 0x00000001 },
+        { 0x0000da,   1, 0x01, 0x00000001 },
+        { 0x0000f8,   1, 0x01, 0x00000003 },
+        { 0x0000fa,   1, 0x01, 0x00000001 },
+        { 0x0000b1,   2, 0x01, 0x00000001 },
+        { 0x00009f,   4, 0x01, 0x0000ffff },
+        { 0x0000a8,   1, 0x01, 0x0000ffff },
+        { 0x0000ad,   1, 0x01, 0x0000013e },
+        { 0x0000e1,   1, 0x01, 0x00000010 },
+        { 0x000290,  16, 0x01, 0x00000000 },
+        { 0x0003b0,  16, 0x01, 0x00000000 },
+        { 0x0002a0,  16, 0x01, 0x00000000 },
+        { 0x000420,  16, 0x01, 0x00000000 },
+        { 0x0002b0,  16, 0x01, 0x00000000 },
+        { 0x000430,  16, 0x01, 0x00000000 },
+        { 0x0002c0,  16, 0x01, 0x00000000 },
+        { 0x0004d0,  16, 0x01, 0x00000000 },
+        { 0x000720,  16, 0x01, 0x00000000 },
+        { 0x0008c0,  16, 0x01, 0x00000000 },
+        { 0x000890,  16, 0x01, 0x00000000 },
+        { 0x0008e0,  16, 0x01, 0x00000000 },
+        { 0x0008a0,  16, 0x01, 0x00000000 },
+        { 0x0008f0,  16, 0x01, 0x00000000 },
+        { 0x00094c,   1, 0x01, 0x000000ff },
+        { 0x00094d,   1, 0x01, 0xffffffff },
+        { 0x00094e,   1, 0x01, 0x00000002 },
+        { 0x0002f2,   2, 0x01, 0x00000001 },
+        { 0x0002f5,   1, 0x01, 0x00000001 },
+        { 0x0002f7,   1, 0x01, 0x00000001 },
+        { 0x000303,   1, 0x01, 0x00000001 },
+        { 0x0002e6,   1, 0x01, 0x00000001 },
+        { 0x000466,   1, 0x01, 0x00000052 },
+        { 0x000301,   1, 0x01, 0x3f800000 },
+        { 0x000304,   1, 0x01, 0x30201000 },
+        { 0x000305,   1, 0x01, 0x70605040 },
+        { 0x000306,   1, 0x01, 0xb8a89888 },
+        { 0x000307,   1, 0x01, 0xf8e8d8c8 },
+        { 0x00030a,   1, 0x01, 0x00ffff00 },
+        { 0x0000de,   1, 0x01, 0x00000001 },
+        { 0x00030b,   1, 0x01, 0x0000001a },
+        { 0x00030c,   1, 0x01, 0x00000001 },
+        { 0x000318,   1, 0x01, 0x00000001 },
+        { 0x000340,   1, 0x01, 0x00000000 },
+        { 0x00037d,   1, 0x01, 0x00000006 },
+        { 0x0003a0,   1, 0x01, 0x00000002 },
+        { 0x0003aa,   1, 0x01, 0x00000001 },
+        { 0x0003a9,   1, 0x01, 0x00000001 },
+        { 0x000380,   1, 0x01, 0x00000001 },
+        { 0x000383,   1, 0x01, 0x00000011 },
+        { 0x000360,   1, 0x01, 0x00000040 },
+        { 0x000366,   2, 0x01, 0x00000000 },
+        { 0x000368,   1, 0x01, 0x00000fff },
+        { 0x000370,   2, 0x01, 0x00000000 },
+        { 0x000372,   1, 0x01, 0x000fffff },
+        { 0x00037a,   1, 0x01, 0x00000012 },
+        { 0x000619,   1, 0x01, 0x00000003 },
+        { 0x000811,   1, 0x01, 0x00000003 },
+        { 0x000812,   1, 0x01, 0x00000004 },
+        { 0x000813,   1, 0x01, 0x00000006 },
+        { 0x000814,   1, 0x01, 0x00000008 },
+        { 0x000815,   1, 0x01, 0x0000000b },
+        { 0x000800,   6, 0x01, 0x00000001 },
+        { 0x000632,   1, 0x01, 0x00000001 },
+        { 0x000633,   1, 0x01, 0x00000002 },
+        { 0x000634,   1, 0x01, 0x00000003 },
+        { 0x000635,   1, 0x01, 0x00000004 },
+        { 0x000654,   1, 0x01, 0x3f800000 },
+        { 0x000657,   1, 0x01, 0x3f800000 },
+        { 0x000655,   2, 0x01, 0x3f800000 },
+        { 0x0006cd,   1, 0x01, 0x3f800000 },
+        { 0x0007f5,   1, 0x01, 0x3f800000 },
+        { 0x0007dc,   1, 0x01, 0x39291909 },
+        { 0x0007dd,   1, 0x01, 0x79695949 },
+        { 0x0007de,   1, 0x01, 0xb9a99989 },
+        { 0x0007df,   1, 0x01, 0xf9e9d9c9 },
+        { 0x0007e8,   1, 0x01, 0x00003210 },
+        { 0x0007e9,   1, 0x01, 0x00007654 },
+        { 0x0007ea,   1, 0x01, 0x00000098 },
+        { 0x0007ec,   1, 0x01, 0x39291909 },
+        { 0x0007ed,   1, 0x01, 0x79695949 },
+        { 0x0007ee,   1, 0x01, 0xb9a99989 },
+        { 0x0007ef,   1, 0x01, 0xf9e9d9c9 },
+        { 0x0007f0,   1, 0x01, 0x00003210 },
+        { 0x0007f1,   1, 0x01, 0x00007654 },
+        { 0x0007f2,   1, 0x01, 0x00000098 },
+        { 0x0005a5,   1, 0x01, 0x00000001 },
+        { 0x0005d0,   1, 0x01, 0x20181008 },
+        { 0x0005d1,   1, 0x01, 0x40383028 },
+        { 0x0005d2,   1, 0x01, 0x60585048 },
+        { 0x0005d3,   1, 0x01, 0x80787068 },
+        { 0x000980, 128, 0x01, 0x00000000 },
+        { 0x000468,   1, 0x01, 0x00000004 },
+        { 0x00046c,   1, 0x01, 0x00000001 },
+        { 0x000470,  96, 0x01, 0x00000000 },
+        { 0x000510,  16, 0x01, 0x3f800000 },
+        { 0x000520,   1, 0x01, 0x000002b6 },
+        { 0x000529,   1, 0x01, 0x00000001 },
+        { 0x000530,  16, 0x01, 0xffff0000 },
+        { 0x000550,  32, 0x01, 0xffff0000 },
+        { 0x000585,   1, 0x01, 0x0000003f },
+        { 0x000576,   1, 0x01, 0x00000003 },
+        { 0x00057b,   1, 0x01, 0x00000059 },
+        { 0x000586,   1, 0x01, 0x00000040 },
+        { 0x000582,   2, 0x01, 0x00000080 },
+        { 0x000595,   1, 0x01, 0x00400040 },
+        { 0x000596,   1, 0x01, 0x00000492 },
+        { 0x000597,   1, 0x01, 0x08080203 },
+        { 0x0005ad,   1, 0x01, 0x00000008 },
+        { 0x000598,   1, 0x01, 0x00020001 },
+        { 0x0005c2,   1, 0x01, 0x00000001 },
+        { 0x000638,   2, 0x01, 0x00000001 },
+        { 0x00063a,   1, 0x01, 0x00000002 },
+        { 0x00063b,   2, 0x01, 0x00000001 },
+        { 0x00063d,   1, 0x01, 0x00000002 },
+        { 0x00063e,   1, 0x01, 0x00000001 },
+        { 0x0008b8,   8, 0x01, 0x00000001 },
+        { 0x000900,   8, 0x01, 0x00000001 },
+        { 0x000908,   8, 0x01, 0x00000002 },
+        { 0x000910,  16, 0x01, 0x00000001 },
+        { 0x000920,   8, 0x01, 0x00000002 },
+        { 0x000928,   8, 0x01, 0x00000001 },
+        { 0x000662,   1, 0x01, 0x00000001 },
+        { 0x000648,   9, 0x01, 0x00000001 },
+        { 0x000658,   1, 0x01, 0x0000000f },
+        { 0x0007ff,   1, 0x01, 0x0000000a },
+        { 0x00066a,   1, 0x01, 0x40000000 },
+        { 0x00066b,   1, 0x01, 0x10000000 },
+        { 0x00066c,   2, 0x01, 0xffff0000 },
+        { 0x0007af,   2, 0x01, 0x00000008 },
+        { 0x0007f6,   1, 0x01, 0x00000001 },
+        { 0x0006b2,   1, 0x01, 0x00000055 },
+        { 0x0007ad,   1, 0x01, 0x00000003 },
+        { 0x000971,   1, 0x01, 0x00000008 },
+        { 0x000972,   1, 0x01, 0x00000040 },
+        { 0x000973,   1, 0x01, 0x0000012c },
+        { 0x00097c,   1, 0x01, 0x00000040 },
+        { 0x000975,   1, 0x01, 0x00000020 },
+        { 0x000976,   1, 0x01, 0x00000001 },
+        { 0x000977,   1, 0x01, 0x00000020 },
+        { 0x000978,   1, 0x01, 0x00000001 },
+        { 0x000957,   1, 0x01, 0x00000003 },
+        { 0x00095e,   1, 0x01, 0x20164010 },
+        { 0x00095f,   1, 0x01, 0x00000020 },
+        { 0x000a0d,   1, 0x01, 0x00000006 },
+        { 0x00097d,   1, 0x01, 0x0000000c },
+        { 0x000683,   1, 0x01, 0x00000006 },
+        { 0x000687,   1, 0x01, 0x003fffff },
+        { 0x0006a0,   1, 0x01, 0x00000005 },
+        { 0x000840,   1, 0x01, 0x00400008 },
+        { 0x000841,   1, 0x01, 0x08000080 },
+        { 0x000842,   1, 0x01, 0x00400008 },
+        { 0x000843,   1, 0x01, 0x08000080 },
+        { 0x000818,   8, 0x01, 0x00000000 },
+        { 0x000848,  16, 0x01, 0x00000000 },
+        { 0x000738,   1, 0x01, 0x00000000 },
+        { 0x0006aa,   1, 0x01, 0x00000001 },
+        { 0x0006ab,   1, 0x01, 0x00000002 },
+        { 0x0006ac,   1, 0x01, 0x00000080 },
+        { 0x0006ad,   2, 0x01, 0x00000100 },
+        { 0x0006b1,   1, 0x01, 0x00000011 },
+        { 0x0006bb,   1, 0x01, 0x000000cf },
+        { 0x0006ce,   1, 0x01, 0x2a712488 },
+        { 0x000739,   1, 0x01, 0x4085c000 },
+        { 0x00073a,   1, 0x01, 0x00000080 },
+        { 0x000786,   1, 0x01, 0x80000100 },
+        { 0x00073c,   1, 0x01, 0x00010100 },
+        { 0x00073d,   1, 0x01, 0x02800000 },
+        { 0x000787,   1, 0x01, 0x000000cf },
+        { 0x00078c,   1, 0x01, 0x00000008 },
+        { 0x000792,   1, 0x01, 0x00000001 },
+        { 0x000794,   3, 0x01, 0x00000001 },
+        { 0x000797,   1, 0x01, 0x000000cf },
+        { 0x000836,   1, 0x01, 0x00000001 },
+        { 0x00079a,   1, 0x01, 0x00000002 },
+        { 0x000833,   1, 0x01, 0x04444480 },
+        { 0x0007a1,   1, 0x01, 0x00000001 },
+        { 0x0007a3,   3, 0x01, 0x00000001 },
+        { 0x000831,   1, 0x01, 0x00000004 },
+        { 0x000b07,   1, 0x01, 0x00000002 },
+        { 0x000b08,   2, 0x01, 0x00000100 },
+        { 0x000b0a,   1, 0x01, 0x00000001 },
+        { 0x000a04,   1, 0x01, 0x000000ff },
+        { 0x000a0b,   1, 0x01, 0x00000040 },
+        { 0x00097f,   1, 0x01, 0x00000100 },
+        { 0x000a02,   1, 0x01, 0x00000001 },
+        { 0x000809,   1, 0x01, 0x00000007 },
+        { 0x00c221,   1, 0x01, 0x00000040 },
+        { 0x00c1b0,   8, 0x01, 0x0000000f },
+        { 0x00c1b8,   1, 0x01, 0x0fac6881 },
+        { 0x00c1b9,   1, 0x01, 0x00fac688 },
+        { 0x00c401,   1, 0x01, 0x00000001 },
+        { 0x00c402,   1, 0x01, 0x00010001 },
+        { 0x00c403,   2, 0x01, 0x00000001 },
+        { 0x00c40e,   1, 0x01, 0x00000020 },
+        { 0x01e100,   1, 0x01, 0x00000001 },
+        { 0x001000,   1, 0x01, 0x00000002 },
+        { 0x0006aa,   1, 0x01, 0x00000001 },
+        { 0x0006ad,   2, 0x01, 0x00000100 },
+        { 0x0006b1,   1, 0x01, 0x00000011 },
+        { 0x00078c,   1, 0x01, 0x00000008 },
+        { 0x000792,   1, 0x01, 0x00000001 },
+        { 0x000794,   3, 0x01, 0x00000001 },
+        { 0x000797,   1, 0x01, 0x000000cf },
+        { 0x00079a,   1, 0x01, 0x00000002 },
+        { 0x0007a1,   1, 0x01, 0x00000001 },
+        { 0x0007a3,   3, 0x01, 0x00000001 },
+        { 0x000831,   1, 0x01, 0x00000004 },
+        { 0x01e100,   1, 0x01, 0x00000001 },
+        { 0x001000,   1, 0x01, 0x00000008 },
+        { 0x000039,   3, 0x01, 0x00000000 },
+        { 0x000380,   1, 0x01, 0x00000001 },
+        { 0x000366,   2, 0x01, 0x00000000 },
+        { 0x000368,   1, 0x01, 0x00000fff },
+        { 0x000370,   2, 0x01, 0x00000000 },
+        { 0x000372,   1, 0x01, 0x000fffff },
+        { 0x000813,   1, 0x01, 0x00000006 },
+        { 0x000814,   1, 0x01, 0x00000008 },
+        { 0x000818,   8, 0x01, 0x00000000 },
+        { 0x000848,  16, 0x01, 0x00000000 },
+        { 0x000738,   1, 0x01, 0x00000000 },
+        { 0x000b07,   1, 0x01, 0x00000002 },
+        { 0x000b08,   2, 0x01, 0x00000100 },
+        { 0x000b0a,   1, 0x01, 0x00000001 },
+        { 0x000a04,   1, 0x01, 0x000000ff },
+        { 0x000a0b,   1, 0x01, 0x00000040 },
+        { 0x00097f,   1, 0x01, 0x00000100 },
+        { 0x000a02,   1, 0x01, 0x00000001 },
+        { 0x000809,   1, 0x01, 0x00000007 },
+        { 0x00c221,   1, 0x01, 0x00000040 },
+        { 0x00c401,   1, 0x01, 0x00000001 },
+        { 0x00c402,   1, 0x01, 0x00010001 },
+        { 0x00c403,   2, 0x01, 0x00000001 },
+        { 0x00c40e,   1, 0x01, 0x00000020 },
+        { 0x01e100,   1, 0x01, 0x00000001 },
+        { 0x001000,   1, 0x01, 0x00000001 },
+        { 0x000b07,   1, 0x01, 0x00000002 },
+        { 0x000b08,   2, 0x01, 0x00000100 },
+        { 0x000b0a,   1, 0x01, 0x00000001 },
+        { 0x01e100,   1, 0x01, 0x00000001 },
+        {}
+};
+
+static const struct nvc0_graph_pack
+gm107_grctx_pack_icmd[] = {
+        { gm107_grctx_init_icmd_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_b097_0[] = {
+        { 0x000800,   8, 0x40, 0x00000000 },
+        { 0x000804,   8, 0x40, 0x00000000 },
+        { 0x000808,   8, 0x40, 0x00000400 },
+        { 0x00080c,   8, 0x40, 0x00000300 },
+        { 0x000810,   1, 0x04, 0x000000cf },
+        { 0x000850,   7, 0x40, 0x00000000 },
+        { 0x000814,   8, 0x40, 0x00000040 },
+        { 0x000818,   8, 0x40, 0x00000001 },
+        { 0x00081c,   8, 0x40, 0x00000000 },
+        { 0x000820,   8, 0x40, 0x00000000 },
+        { 0x001c00,  16, 0x10, 0x00000000 },
+        { 0x001c04,  16, 0x10, 0x00000000 },
+        { 0x001c08,  16, 0x10, 0x00000000 },
+        { 0x001c0c,  16, 0x10, 0x00000000 },
+        { 0x001d00,  16, 0x10, 0x00000000 },
+        { 0x001d04,  16, 0x10, 0x00000000 },
+        { 0x001d08,  16, 0x10, 0x00000000 },
+        { 0x001d0c,  16, 0x10, 0x00000000 },
+        { 0x001f00,  16, 0x08, 0x00000000 },
+        { 0x001f04,  16, 0x08, 0x00000000 },
+        { 0x001f80,  16, 0x08, 0x00000000 },
+        { 0x001f84,  16, 0x08, 0x00000000 },
+        { 0x002000,   1, 0x04, 0x00000000 },
+        { 0x002040,   1, 0x04, 0x00000011 },
+        { 0x002080,   1, 0x04, 0x00000020 },
+        { 0x0020c0,   1, 0x04, 0x00000030 },
+        { 0x002100,   1, 0x04, 0x00000040 },
+        { 0x002140,   1, 0x04, 0x00000051 },
+        { 0x00200c,   6, 0x40, 0x00000001 },
+        { 0x002010,   1, 0x04, 0x00000000 },
+        { 0x002050,   1, 0x04, 0x00000000 },
+        { 0x002090,   1, 0x04, 0x00000001 },
+        { 0x0020d0,   1, 0x04, 0x00000002 },
+        { 0x002110,   1, 0x04, 0x00000003 },
+        { 0x002150,   1, 0x04, 0x00000004 },
+        { 0x000380,   4, 0x20, 0x00000000 },
+        { 0x000384,   4, 0x20, 0x00000000 },
+        { 0x000388,   4, 0x20, 0x00000000 },
+        { 0x00038c,   4, 0x20, 0x00000000 },
+        { 0x000700,   4, 0x10, 0x00000000 },
+        { 0x000704,   4, 0x10, 0x00000000 },
+        { 0x000708,   4, 0x10, 0x00000000 },
+        { 0x002800, 128, 0x04, 0x00000000 },
+        { 0x000a00,  16, 0x20, 0x00000000 },
+        { 0x000a04,  16, 0x20, 0x00000000 },
+        { 0x000a08,  16, 0x20, 0x00000000 },
+        { 0x000a0c,  16, 0x20, 0x00000000 },
+        { 0x000a10,  16, 0x20, 0x00000000 },
+        { 0x000a14,  16, 0x20, 0x00000000 },
+        { 0x000c00,  16, 0x10, 0x00000000 },
+        { 0x000c04,  16, 0x10, 0x00000000 },
+        { 0x000c08,  16, 0x10, 0x00000000 },
+        { 0x000c0c,  16, 0x10, 0x3f800000 },
+        { 0x000d00,   8, 0x08, 0xffff0000 },
+        { 0x000d04,   8, 0x08, 0xffff0000 },
+        { 0x000e00,  16, 0x10, 0x00000000 },
+        { 0x000e04,  16, 0x10, 0xffff0000 },
+        { 0x000e08,  16, 0x10, 0xffff0000 },
+        { 0x000d40,   4, 0x08, 0x00000000 },
+        { 0x000d44,   4, 0x08, 0x00000000 },
+        { 0x001e00,   8, 0x20, 0x00000001 },
+        { 0x001e04,   8, 0x20, 0x00000001 },
+        { 0x001e08,   8, 0x20, 0x00000002 },
+        { 0x001e0c,   8, 0x20, 0x00000001 },
+        { 0x001e10,   8, 0x20, 0x00000001 },
+        { 0x001e14,   8, 0x20, 0x00000002 },
+        { 0x001e18,   8, 0x20, 0x00000001 },
+        { 0x001480,   8, 0x10, 0x00000000 },
+        { 0x001484,   8, 0x10, 0x00000000 },
+        { 0x001488,   8, 0x10, 0x00000000 },
+        { 0x003400, 128, 0x04, 0x00000000 },
+        { 0x00030c,   1, 0x04, 0x00000001 },
+        { 0x001944,   1, 0x04, 0x00000000 },
+        { 0x001514,   1, 0x04, 0x00000000 },
+        { 0x000d68,   1, 0x04, 0x0000ffff },
+        { 0x00121c,   1, 0x04, 0x0fac6881 },
+        { 0x000fac,   1, 0x04, 0x00000001 },
+        { 0x001538,   1, 0x04, 0x00000001 },
+        { 0x000fe0,   2, 0x04, 0x00000000 },
+        { 0x000fe8,   1, 0x04, 0x00000014 },
+        { 0x000fec,   1, 0x04, 0x00000040 },
+        { 0x000ff0,   1, 0x04, 0x00000000 },
+        { 0x00179c,   1, 0x04, 0x00000000 },
+        { 0x001228,   1, 0x04, 0x00000400 },
+        { 0x00122c,   1, 0x04, 0x00000300 },
+        { 0x001230,   1, 0x04, 0x00010001 },
+        { 0x0007f8,   1, 0x04, 0x00000000 },
+        { 0x0015b4,   1, 0x04, 0x00000001 },
+        { 0x0015cc,   1, 0x04, 0x00000000 },
+        { 0x001534,   1, 0x04, 0x00000000 },
+        { 0x000754,   1, 0x04, 0x00000001 },
+        { 0x000fb0,   1, 0x04, 0x00000000 },
+        { 0x0015d0,   1, 0x04, 0x00000000 },
+        { 0x00153c,   1, 0x04, 0x00000000 },
+        { 0x0016b4,   1, 0x04, 0x00000003 },
+        { 0x000fbc,   4, 0x04, 0x0000ffff },
+        { 0x000df8,   2, 0x04, 0x00000000 },
+        { 0x001948,   1, 0x04, 0x00000000 },
+        { 0x001970,   1, 0x04, 0x00000001 },
+        { 0x00161c,   1, 0x04, 0x000009f0 },
+        { 0x000dcc,   1, 0x04, 0x00000010 },
+        { 0x0015e4,   1, 0x04, 0x00000000 },
+        { 0x001160,  32, 0x04, 0x25e00040 },
+        { 0x001880,  32, 0x04, 0x00000000 },
+        { 0x000f84,   2, 0x04, 0x00000000 },
+        { 0x0017c8,   2, 0x04, 0x00000000 },
+        { 0x0017d0,   1, 0x04, 0x000000ff },
+        { 0x0017d4,   1, 0x04, 0xffffffff },
+        { 0x0017d8,   1, 0x04, 0x00000002 },
+        { 0x0017dc,   1, 0x04, 0x00000000 },
+        { 0x0015f4,   2, 0x04, 0x00000000 },
+        { 0x001434,   2, 0x04, 0x00000000 },
+        { 0x000d74,   1, 0x04, 0x00000000 },
+        { 0x0013a4,   1, 0x04, 0x00000000 },
+        { 0x001318,   1, 0x04, 0x00000001 },
+        { 0x001080,   2, 0x04, 0x00000000 },
+        { 0x001088,   2, 0x04, 0x00000001 },
+        { 0x001090,   1, 0x04, 0x00000000 },
+        { 0x001094,   1, 0x04, 0x00000001 },
+        { 0x001098,   1, 0x04, 0x00000000 },
+        { 0x00109c,   1, 0x04, 0x00000001 },
+        { 0x0010a0,   2, 0x04, 0x00000000 },
+        { 0x001644,   1, 0x04, 0x00000000 },
+        { 0x000748,   1, 0x04, 0x00000000 },
+        { 0x000de8,   1, 0x04, 0x00000000 },
+        { 0x001648,   1, 0x04, 0x00000000 },
+        { 0x0012a4,   1, 0x04, 0x00000000 },
+        { 0x001120,   4, 0x04, 0x00000000 },
+        { 0x001118,   1, 0x04, 0x00000000 },
+        { 0x00164c,   1, 0x04, 0x00000000 },
+        { 0x001658,   1, 0x04, 0x00000000 },
+        { 0x001910,   1, 0x04, 0x00000290 },
+        { 0x001518,   1, 0x04, 0x00000000 },
+        { 0x00165c,   1, 0x04, 0x00000001 },
+        { 0x001520,   1, 0x04, 0x00000000 },
+        { 0x001604,   1, 0x04, 0x00000000 },
+        { 0x001570,   1, 0x04, 0x00000000 },
+        { 0x0013b0,   2, 0x04, 0x3f800000 },
+        { 0x00020c,   1, 0x04, 0x00000000 },
+        { 0x001670,   1, 0x04, 0x30201000 },
+        { 0x001674,   1, 0x04, 0x70605040 },
+        { 0x001678,   1, 0x04, 0xb8a89888 },
+        { 0x00167c,   1, 0x04, 0xf8e8d8c8 },
+        { 0x00166c,   1, 0x04, 0x00000000 },
+        { 0x001680,   1, 0x04, 0x00ffff00 },
+        { 0x0012d0,   1, 0x04, 0x00000003 },
+        { 0x0012d4,   1, 0x04, 0x00000002 },
+        { 0x001684,   2, 0x04, 0x00000000 },
+        { 0x000dac,   2, 0x04, 0x00001b02 },
+        { 0x000db4,   1, 0x04, 0x00000000 },
+        { 0x00168c,   1, 0x04, 0x00000000 },
+        { 0x0015bc,   1, 0x04, 0x00000000 },
+        { 0x00156c,   1, 0x04, 0x00000000 },
+        { 0x00187c,   1, 0x04, 0x00000000 },
+        { 0x001110,   1, 0x04, 0x00000001 },
+        { 0x000dc0,   3, 0x04, 0x00000000 },
+        { 0x000f40,   5, 0x04, 0x00000000 },
+        { 0x001234,   1, 0x04, 0x00000000 },
+        { 0x001690,   1, 0x04, 0x00000000 },
+        { 0x000790,   5, 0x04, 0x00000000 },
+        { 0x00077c,   1, 0x04, 0x00000000 },
+        { 0x001000,   1, 0x04, 0x00000010 },
+        { 0x0010fc,   1, 0x04, 0x00000000 },
+        { 0x001290,   1, 0x04, 0x00000000 },
+        { 0x000218,   1, 0x04, 0x00000010 },
+        { 0x0012d8,   1, 0x04, 0x00000000 },
+        { 0x0012dc,   1, 0x04, 0x00000010 },
+        { 0x000d94,   1, 0x04, 0x00000001 },
+        { 0x00155c,   2, 0x04, 0x00000000 },
+        { 0x001564,   1, 0x04, 0x00000fff },
+        { 0x001574,   2, 0x04, 0x00000000 },
+        { 0x00157c,   1, 0x04, 0x000fffff },
+        { 0x001354,   1, 0x04, 0x00000000 },
+        { 0x001610,   1, 0x04, 0x00000012 },
+        { 0x001608,   2, 0x04, 0x00000000 },
+        { 0x00260c,   1, 0x04, 0x00000000 },
+        { 0x0007ac,   1, 0x04, 0x00000000 },
+        { 0x00162c,   1, 0x04, 0x00000003 },
+        { 0x000210,   1, 0x04, 0x00000000 },
+        { 0x000320,   1, 0x04, 0x00000000 },
+        { 0x000324,   6, 0x04, 0x3f800000 },
+        { 0x000750,   1, 0x04, 0x00000000 },
+        { 0x000760,   1, 0x04, 0x39291909 },
+        { 0x000764,   1, 0x04, 0x79695949 },
+        { 0x000768,   1, 0x04, 0xb9a99989 },
+        { 0x00076c,   1, 0x04, 0xf9e9d9c9 },
+        { 0x000770,   1, 0x04, 0x30201000 },
+        { 0x000774,   1, 0x04, 0x70605040 },
+        { 0x000778,   1, 0x04, 0x00009080 },
+        { 0x000780,   1, 0x04, 0x39291909 },
+        { 0x000784,   1, 0x04, 0x79695949 },
+        { 0x000788,   1, 0x04, 0xb9a99989 },
+        { 0x00078c,   1, 0x04, 0xf9e9d9c9 },
+        { 0x0007d0,   1, 0x04, 0x30201000 },
+        { 0x0007d4,   1, 0x04, 0x70605040 },
+        { 0x0007d8,   1, 0x04, 0x00009080 },
+        { 0x00037c,   1, 0x04, 0x00000001 },
+        { 0x000740,   2, 0x04, 0x00000000 },
+        { 0x002600,   1, 0x04, 0x00000000 },
+        { 0x001918,   1, 0x04, 0x00000000 },
+        { 0x00191c,   1, 0x04, 0x00000900 },
+        { 0x001920,   1, 0x04, 0x00000405 },
+        { 0x001308,   1, 0x04, 0x00000001 },
+        { 0x001924,   1, 0x04, 0x00000000 },
+        { 0x0013ac,   1, 0x04, 0x00000000 },
+        { 0x00192c,   1, 0x04, 0x00000001 },
+        { 0x00193c,   1, 0x04, 0x00002c1c },
+        { 0x000d7c,   1, 0x04, 0x00000000 },
+        { 0x000f8c,   1, 0x04, 0x00000000 },
+        { 0x0002c0,   1, 0x04, 0x00000001 },
+        { 0x001510,   1, 0x04, 0x00000000 },
+        { 0x001940,   1, 0x04, 0x00000000 },
+        { 0x000ff4,   2, 0x04, 0x00000000 },
+        { 0x00194c,   2, 0x04, 0x00000000 },
+        { 0x001968,   1, 0x04, 0x00000000 },
+        { 0x001590,   1, 0x04, 0x0000003f },
+        { 0x0007e8,   4, 0x04, 0x00000000 },
+        { 0x00196c,   1, 0x04, 0x00000011 },
+        { 0x0002e4,   1, 0x04, 0x0000b001 },
+        { 0x00036c,   2, 0x04, 0x00000000 },
+        { 0x00197c,   1, 0x04, 0x00000000 },
+        { 0x000fcc,   2, 0x04, 0x00000000 },
+        { 0x0002d8,   1, 0x04, 0x00000040 },
+        { 0x001980,   1, 0x04, 0x00000080 },
+        { 0x001504,   1, 0x04, 0x00000080 },
+        { 0x001984,   1, 0x04, 0x00000000 },
+        { 0x000f60,   1, 0x04, 0x00000000 },
+        { 0x000f64,   1, 0x04, 0x00400040 },
+        { 0x000f68,   1, 0x04, 0x00002212 },
+        { 0x000f6c,   1, 0x04, 0x08080203 },
+        { 0x001108,   1, 0x04, 0x00000008 },
+        { 0x000f70,   1, 0x04, 0x00080001 },
+        { 0x000ffc,   1, 0x04, 0x00000000 },
+        { 0x000300,   1, 0x04, 0x00000001 },
+        { 0x0013a8,   1, 0x04, 0x00000000 },
+        { 0x0012ec,   1, 0x04, 0x00000000 },
+        { 0x001310,   1, 0x04, 0x00000000 },
+        { 0x001314,   1, 0x04, 0x00000001 },
+        { 0x001380,   1, 0x04, 0x00000000 },
+        { 0x001384,   4, 0x04, 0x00000001 },
+        { 0x001394,   1, 0x04, 0x00000000 },
+        { 0x00139c,   1, 0x04, 0x00000000 },
+        { 0x001398,   1, 0x04, 0x00000000 },
+        { 0x001594,   1, 0x04, 0x00000000 },
+        { 0x001598,   4, 0x04, 0x00000001 },
+        { 0x000f54,   3, 0x04, 0x00000000 },
+        { 0x0019bc,   1, 0x04, 0x00000000 },
+        { 0x000f9c,   2, 0x04, 0x00000000 },
+        { 0x0012cc,   1, 0x04, 0x00000000 },
+        { 0x0012e8,   1, 0x04, 0x00000000 },
+        { 0x00130c,   1, 0x04, 0x00000001 },
+        { 0x001360,   8, 0x04, 0x00000000 },
+        { 0x00133c,   2, 0x04, 0x00000001 },
+        { 0x001344,   1, 0x04, 0x00000002 },
+        { 0x001348,   2, 0x04, 0x00000001 },
+        { 0x001350,   1, 0x04, 0x00000002 },
+        { 0x001358,   1, 0x04, 0x00000001 },
+        { 0x0012e4,   1, 0x04, 0x00000000 },
+        { 0x00131c,   4, 0x04, 0x00000000 },
+        { 0x0019c0,   1, 0x04, 0x00000000 },
+        { 0x001140,   1, 0x04, 0x00000000 },
+        { 0x000dd0,   1, 0x04, 0x00000000 },
+        { 0x000dd4,   1, 0x04, 0x00000001 },
+        { 0x0002f4,   1, 0x04, 0x00000000 },
+        { 0x0019c4,   1, 0x04, 0x00000000 },
+        { 0x0019c8,   1, 0x04, 0x00001500 },
+        { 0x00135c,   1, 0x04, 0x00000000 },
+        { 0x000f90,   1, 0x04, 0x00000000 },
+        { 0x0019e0,   8, 0x04, 0x00000001 },
+        { 0x0019cc,   1, 0x04, 0x00000001 },
+        { 0x0015b8,   1, 0x04, 0x00000000 },
+        { 0x001a00,   1, 0x04, 0x00001111 },
+        { 0x001a04,   7, 0x04, 0x00000000 },
+        { 0x000d6c,   2, 0x04, 0xffff0000 },
+        { 0x0010f8,   1, 0x04, 0x00001010 },
+        { 0x000d80,   5, 0x04, 0x00000000 },
+        { 0x000da0,   1, 0x04, 0x00000000 },
+        { 0x0007a4,   2, 0x04, 0x00000000 },
+        { 0x001508,   1, 0x04, 0x80000000 },
+        { 0x00150c,   1, 0x04, 0x40000000 },
+        { 0x001668,   1, 0x04, 0x00000000 },
+        { 0x000318,   2, 0x04, 0x00000008 },
+        { 0x000d9c,   1, 0x04, 0x00000001 },
+        { 0x000f14,   1, 0x04, 0x00000000 },
+        { 0x000374,   1, 0x04, 0x00000000 },
+        { 0x000378,   1, 0x04, 0x0000000c },
+        { 0x0007dc,   1, 0x04, 0x00000000 },
+        { 0x00074c,   1, 0x04, 0x00000055 },
+        { 0x001420,   1, 0x04, 0x00000003 },
+        { 0x001008,   1, 0x04, 0x00000008 },
+        { 0x00100c,   1, 0x04, 0x00000040 },
+        { 0x001010,   1, 0x04, 0x0000012c },
+        { 0x000d60,   1, 0x04, 0x00000040 },
+        { 0x001018,   1, 0x04, 0x00000020 },
+        { 0x00101c,   1, 0x04, 0x00000001 },
+        { 0x001020,   1, 0x04, 0x00000020 },
+        { 0x001024,   1, 0x04, 0x00000001 },
+        { 0x001444,   3, 0x04, 0x00000000 },
+        { 0x000360,   1, 0x04, 0x20164010 },
+        { 0x000364,   1, 0x04, 0x00000020 },
+        { 0x000368,   1, 0x04, 0x00000000 },
+        { 0x000da8,   1, 0x04, 0x00000030 },
+        { 0x000de4,   1, 0x04, 0x00000000 },
+        { 0x000204,   1, 0x04, 0x00000006 },
+        { 0x0002d0,   1, 0x04, 0x003fffff },
+        { 0x001220,   1, 0x04, 0x00000005 },
+        { 0x000fdc,   1, 0x04, 0x00000000 },
+        { 0x000f98,   1, 0x04, 0x00400008 },
+        { 0x001284,   1, 0x04, 0x08000080 },
+        { 0x001450,   1, 0x04, 0x00400008 },
+        { 0x001454,   1, 0x04, 0x08000080 },
+        { 0x000214,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_pack
+gm107_grctx_pack_mthd[] = {
+        { gm107_grctx_init_b097_0, 0xb097 },
+        { nvc0_grctx_init_902d_0, 0x902d },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_fe_0[] = {
+        { 0x404004,   8, 0x04, 0x00000000 },
+        { 0x404024,   1, 0x04, 0x0000e000 },
+        { 0x404028,   8, 0x04, 0x00000000 },
+        { 0x4040a8,   8, 0x04, 0x00000000 },
+        { 0x4040c8,   1, 0x04, 0xf800008f },
+        { 0x4040d0,   6, 0x04, 0x00000000 },
+        { 0x4040f8,   1, 0x04, 0x00000000 },
+        { 0x404100,  10, 0x04, 0x00000000 },
+        { 0x404130,   2, 0x04, 0x00000000 },
+        { 0x404150,   1, 0x04, 0x0000002e },
+        { 0x404154,   1, 0x04, 0x00000400 },
+        { 0x404158,   1, 0x04, 0x00000200 },
+        { 0x404164,   1, 0x04, 0x00000045 },
+        { 0x40417c,   2, 0x04, 0x00000000 },
+        { 0x404194,   1, 0x04, 0x01000700 },
+        { 0x4041a0,   4, 0x04, 0x00000000 },
+        { 0x404200,   4, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_ds_0[] = {
+        { 0x405800,   1, 0x04, 0x0f8001bf },
+        { 0x405830,   1, 0x04, 0x0aa01000 },
+        { 0x405834,   1, 0x04, 0x08000000 },
+        { 0x405838,   1, 0x04, 0x00000000 },
+        { 0x405854,   1, 0x04, 0x00000000 },
+        { 0x405870,   4, 0x04, 0x00000001 },
+        { 0x405a00,   2, 0x04, 0x00000000 },
+        { 0x405a18,   1, 0x04, 0x00000000 },
+        { 0x405a1c,   1, 0x04, 0x000000ff },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_pd_0[] = {
+        { 0x406020,   1, 0x04, 0x07410001 },
+        { 0x406028,   4, 0x04, 0x00000001 },
+        { 0x4064a8,   1, 0x04, 0x00000000 },
+        { 0x4064ac,   1, 0x04, 0x00003fff },
+        { 0x4064b0,   3, 0x04, 0x00000000 },
+        { 0x4064c0,   1, 0x04, 0x80400280 },
+        { 0x4064c4,   1, 0x04, 0x0400ffff },
+        { 0x4064c8,   1, 0x04, 0x018001ff },
+        { 0x4064cc,   9, 0x04, 0x00000000 },
+        { 0x4064fc,   1, 0x04, 0x0000022a },
+        { 0x406500,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_be_0[] = {
+        { 0x408800,   1, 0x04, 0x32802a3c },
+        { 0x408804,   1, 0x04, 0x00000040 },
+        { 0x408808,   1, 0x04, 0x1003e005 },
+        { 0x408840,   1, 0x04, 0x0000000b },
+        { 0x408900,   1, 0x04, 0xb080b801 },
+        { 0x408904,   1, 0x04, 0x63038001 },
+        { 0x408908,   1, 0x04, 0x02c8102f },
+        { 0x408980,   1, 0x04, 0x0000011d },
+        {}
+};
+
+static const struct nvc0_graph_pack
+gm107_grctx_pack_hub[] = {
+        { nvc0_grctx_init_main_0 },
+        { gm107_grctx_init_fe_0 },
+        { nvf0_grctx_init_pri_0 },
+        { nve4_grctx_init_memfmt_0 },
+        { gm107_grctx_init_ds_0 },
+        { nvf0_grctx_init_cwd_0 },
+        { gm107_grctx_init_pd_0 },
+        { nv108_grctx_init_rstr2d_0 },
+        { nve4_grctx_init_scc_0 },
+        { gm107_grctx_init_be_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_gpc_unk_0[] = {
+        { 0x418380,   1, 0x04, 0x00000056 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_gpc_unk_1[] = {
+        { 0x418600,   1, 0x04, 0x0000007f },
+        { 0x418684,   1, 0x04, 0x0000001f },
+        { 0x418700,   1, 0x04, 0x00000002 },
+        { 0x418704,   1, 0x04, 0x00000080 },
+        { 0x418708,   1, 0x04, 0x40000000 },
+        { 0x41870c,   2, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_setup_0[] = {
+        { 0x418800,   1, 0x04, 0x7006863a },
+        { 0x418810,   1, 0x04, 0x00000000 },
+        { 0x418828,   1, 0x04, 0x00000044 },
+        { 0x418830,   1, 0x04, 0x10000001 },
+        { 0x4188d8,   1, 0x04, 0x00000008 },
+        { 0x4188e0,   1, 0x04, 0x01000000 },
+        { 0x4188e8,   5, 0x04, 0x00000000 },
+        { 0x4188fc,   1, 0x04, 0x20100058 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_gpc_unk_2[] = {
+        { 0x418d24,   1, 0x04, 0x00000000 },
+        { 0x418e00,   1, 0x04, 0x90000000 },
+        { 0x418e24,   1, 0x04, 0x00000000 },
+        { 0x418e28,   1, 0x04, 0x00000030 },
+        { 0x418e30,   1, 0x04, 0x00000000 },
+        { 0x418e34,   1, 0x04, 0x00010000 },
+        { 0x418e38,   1, 0x04, 0x00000000 },
+        { 0x418e40,  22, 0x04, 0x00000000 },
+        { 0x418ea0,   2, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_pack
+gm107_grctx_pack_gpc[] = {
+        { gm107_grctx_init_gpc_unk_0 },
+        { nv108_grctx_init_prop_0 },
+        { gm107_grctx_init_gpc_unk_1 },
+        { gm107_grctx_init_setup_0 },
+        { nvc0_grctx_init_zcull_0 },
+        { nv108_grctx_init_crstr_0 },
+        { nve4_grctx_init_gpm_0 },
+        { gm107_grctx_init_gpc_unk_2 },
+        { nvc0_grctx_init_gcc_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_tex_0[] = {
+        { 0x419a00,   1, 0x04, 0x000300f0 },
+        { 0x419a04,   1, 0x04, 0x00000005 },
+        { 0x419a08,   1, 0x04, 0x00000421 },
+        { 0x419a0c,   1, 0x04, 0x00120000 },
+        { 0x419a10,   1, 0x04, 0x00000000 },
+        { 0x419a14,   1, 0x04, 0x00002200 },
+        { 0x419a1c,   1, 0x04, 0x0000c000 },
+        { 0x419a20,   1, 0x04, 0x20008a00 },
+        { 0x419a30,   1, 0x04, 0x00000001 },
+        { 0x419a3c,   1, 0x04, 0x00000002 },
+        { 0x419ac4,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_mpc_0[] = {
+        { 0x419c00,   1, 0x04, 0x0000001a },
+        { 0x419c04,   1, 0x04, 0x80000006 },
+        { 0x419c08,   1, 0x04, 0x00000002 },
+        { 0x419c20,   1, 0x04, 0x00000000 },
+        { 0x419c24,   1, 0x04, 0x00084210 },
+        { 0x419c28,   1, 0x04, 0x3efbefbe },
+        { 0x419c2c,   1, 0x04, 0x00000000 },
+        { 0x419c34,   1, 0x04, 0x01ff1ff3 },
+        { 0x419c3c,   1, 0x04, 0x00001919 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_l1c_0[] = {
+        { 0x419c84,   1, 0x04, 0x00000020 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_sm_0[] = {
+        { 0x419e04,   3, 0x04, 0x00000000 },
+        { 0x419e10,   1, 0x04, 0x00001c02 },
+        { 0x419e44,   1, 0x04, 0x00d3eff2 },
+        { 0x419e48,   1, 0x04, 0x00000000 },
+        { 0x419e4c,   1, 0x04, 0x0000007f },
+        { 0x419e50,   1, 0x04, 0x00000000 },
+        { 0x419e60,   4, 0x04, 0x00000000 },
+        { 0x419e74,  10, 0x04, 0x00000000 },
+        { 0x419eac,   1, 0x04, 0x0001cf8b },
+        { 0x419eb0,   1, 0x04, 0x00030300 },
+        { 0x419eb8,   1, 0x04, 0x00000000 },
+        { 0x419ef0,  24, 0x04, 0x00000000 },
+        { 0x419f68,   2, 0x04, 0x00000000 },
+        { 0x419f70,   1, 0x04, 0x00000020 },
+        { 0x419f78,   1, 0x04, 0x000003eb },
+        { 0x419f7c,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_pack
+gm107_grctx_pack_tpc[] = {
+        { nvd7_grctx_init_pe_0 },
+        { gm107_grctx_init_tex_0 },
+        { gm107_grctx_init_mpc_0 },
+        { gm107_grctx_init_l1c_0 },
+        { gm107_grctx_init_sm_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_cbm_0[] = {
+        { 0x41bec0,   1, 0x04, 0x00000000 },
+        { 0x41bec4,   1, 0x04, 0x01050000 },
+        { 0x41bee4,   1, 0x04, 0x00000000 },
+        { 0x41bef0,   1, 0x04, 0x000003ff },
+        { 0x41bef4,   2, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_grctx_init_wwdx_0[] = {
+        { 0x41bf00,   1, 0x04, 0x0a418820 },
+        { 0x41bf04,   1, 0x04, 0x062080e6 },
+        { 0x41bf08,   1, 0x04, 0x020398a4 },
+        { 0x41bf0c,   1, 0x04, 0x0e629062 },
+        { 0x41bf10,   1, 0x04, 0x0a418820 },
+        { 0x41bf14,   1, 0x04, 0x000000e6 },
+        { 0x41bfd0,   1, 0x04, 0x00900103 },
+        { 0x41bfe0,   1, 0x04, 0x80000000 },
+        { 0x41bfe4,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_pack
+gm107_grctx_pack_ppc[] = {
+        { nve4_grctx_init_pes_0 },
+        { gm107_grctx_init_cbm_0 },
+        { gm107_grctx_init_wwdx_0 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH context implementation
+ ******************************************************************************/
+
+static void
+gm107_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
+{
+        mmio_data(0x003000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
+        mmio_data(0x008000, 0x0100, NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS);
+        mmio_data(0x060000, 0x1000, NV_MEM_ACCESS_RW);
+
+        mmio_list(0x40800c, 0x00000000,  8, 1);
+        mmio_list(0x408010, 0x80000000,  0, 0);
+        mmio_list(0x419004, 0x00000000,  8, 1);
+        mmio_list(0x419008, 0x00000000,  0, 0);
+        mmio_list(0x4064cc, 0x80000000,  0, 0);
+        mmio_list(0x418e30, 0x80000000,  0, 0);
+
+        mmio_list(0x408004, 0x00000000,  8, 0);
+        mmio_list(0x408008, 0x80000030,  0, 0);
+        mmio_list(0x418e24, 0x00000000,  8, 0);
+        mmio_list(0x418e28, 0x80000030,  0, 0);
+
+        mmio_list(0x418810, 0x80000000, 12, 2);
+        mmio_list(0x419848, 0x10000000, 12, 2);
+        mmio_list(0x419c2c, 0x10000000, 12, 2);
+
+        mmio_list(0x405830, 0x0aa01000,  0, 0);
+        mmio_list(0x4064c4, 0x0400ffff,  0, 0);
+
+        /*XXX*/
+        mmio_list(0x5030c0, 0x00001540,  0, 0);
+        mmio_list(0x5030f4, 0x00000000,  0, 0);
+        mmio_list(0x5030e4, 0x00002000,  0, 0);
+        mmio_list(0x5030f8, 0x00003fc0,  0, 0);
+        mmio_list(0x418ea0, 0x07151540,  0, 0);
+
+        mmio_list(0x5032c0, 0x00001540,  0, 0);
+        mmio_list(0x5032f4, 0x00001fe0,  0, 0);
+        mmio_list(0x5032e4, 0x00002000,  0, 0);
+        mmio_list(0x5032f8, 0x00006fc0,  0, 0);
+        mmio_list(0x418ea4, 0x07151540,  0, 0);
+}
+
+static void
+gm107_grctx_generate_tpcid(struct nvc0_graph_priv *priv)
+{
+        int gpc, tpc, id;
+
+        for (tpc = 0, id = 0; tpc < 4; tpc++) {
+                for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+                        if (tpc < priv->tpc_nr[gpc]) {
+                                nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x698), id);
+                                nv_wr32(priv, GPC_UNIT(gpc, 0x0c10 + tpc * 4), id);
+                                nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x088), id);
+                                id++;
+                        }
+
+                        nv_wr32(priv, GPC_UNIT(gpc, 0x0c08), priv->tpc_nr[gpc]);
+                        nv_wr32(priv, GPC_UNIT(gpc, 0x0c8c), priv->tpc_nr[gpc]);
+                }
+        }
+}
+
+static void
+gm107_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
+{
+        struct nvc0_grctx_oclass *oclass = (void *)nv_engine(priv)->cclass;
+        int i;
+
+        nvc0_graph_mmio(priv, oclass->hub);
+        nvc0_graph_mmio(priv, oclass->gpc);
+        nvc0_graph_mmio(priv, oclass->zcull);
+        nvc0_graph_mmio(priv, oclass->tpc);
+        nvc0_graph_mmio(priv, oclass->ppc);
+
+        nv_wr32(priv, 0x404154, 0x00000000);
+
+        oclass->mods(priv, info);
+        oclass->unkn(priv);
+
+        gm107_grctx_generate_tpcid(priv);
+        nvc0_grctx_generate_r406028(priv);
+        nve4_grctx_generate_r418bb8(priv);
+        nvc0_grctx_generate_r406800(priv);
+
+        nv_wr32(priv, 0x4064d0, 0x00000001);
+        for (i = 1; i < 8; i++)
+                nv_wr32(priv, 0x4064d0 + (i * 0x04), 0x00000000);
+        nv_wr32(priv, 0x406500, 0x00000001);
+
+        nv_wr32(priv, 0x405b00, (priv->tpc_total << 8) | priv->gpc_nr);
+
+        if (priv->gpc_nr == 1) {
+                nv_mask(priv, 0x408850, 0x0000000f, priv->tpc_nr[0]);
+                nv_mask(priv, 0x408958, 0x0000000f, priv->tpc_nr[0]);
+        } else {
+                nv_mask(priv, 0x408850, 0x0000000f, priv->gpc_nr);
+                nv_mask(priv, 0x408958, 0x0000000f, priv->gpc_nr);
+        }
+
+        nvc0_graph_icmd(priv, oclass->icmd);
+        nv_wr32(priv, 0x404154, 0x00000400);
+        nvc0_graph_mthd(priv, oclass->mthd);
+
+        nv_mask(priv, 0x419e00, 0x00808080, 0x00808080);
+        nv_mask(priv, 0x419ccc, 0x80000000, 0x80000000);
+        nv_mask(priv, 0x419f80, 0x80000000, 0x80000000);
+        nv_mask(priv, 0x419f88, 0x80000000, 0x80000000);
+}
+
+struct nouveau_oclass *
+gm107_grctx_oclass = &(struct nvc0_grctx_oclass) {
+        .base.handle = NV_ENGCTX(GR, 0x08),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nvc0_graph_context_ctor,
+                .dtor = nvc0_graph_context_dtor,
+                .init = _nouveau_graph_context_init,
+                .fini = _nouveau_graph_context_fini,
+                .rd32 = _nouveau_graph_context_rd32,
+                .wr32 = _nouveau_graph_context_wr32,
+        },
+        .main  = gm107_grctx_generate_main,
+        .mods  = gm107_grctx_generate_mods,
+        .unkn  = nve4_grctx_generate_unkn,
+        .hub   = gm107_grctx_pack_hub,
+        .gpc   = gm107_grctx_pack_gpc,
+        .zcull = nvc0_grctx_pack_zcull,
+        .tpc   = gm107_grctx_pack_tpc,
+        .ppc   = gm107_grctx_pack_ppc,
+        .icmd  = gm107_grctx_pack_icmd,
+        .mthd  = gm107_grctx_pack_mthd,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnv108.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnv108.c
index a86bd3352bf8..48351b4d6d6b 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnv108.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnv108.c
@@ -22,10 +22,14 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
-#include "nvc0.h"
+#include "ctxnvc0.h"
 
-static struct nvc0_graph_init
-nv108_grctx_init_icmd[] = {
+/*******************************************************************************
+ * PGRAPH context register lists
+ ******************************************************************************/
+
+static const struct nvc0_graph_init
+nv108_grctx_init_icmd_0[] = {
         { 0x001000,   1, 0x01, 0x00000004 },
         { 0x000039,   3, 0x01, 0x00000000 },
         { 0x0000a9,   1, 0x01, 0x0000ffff },
@@ -274,839 +278,14 @@ nv108_grctx_init_icmd[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nv108_grctx_init_a197[] = {
-        { 0x000800,   1, 0x04, 0x00000000 },
-        { 0x000840,   1, 0x04, 0x00000000 },
-        { 0x000880,   1, 0x04, 0x00000000 },
-        { 0x0008c0,   1, 0x04, 0x00000000 },
-        { 0x000900,   1, 0x04, 0x00000000 },
-        { 0x000940,   1, 0x04, 0x00000000 },
-        { 0x000980,   1, 0x04, 0x00000000 },
-        { 0x0009c0,   1, 0x04, 0x00000000 },
-        { 0x000804,   1, 0x04, 0x00000000 },
-        { 0x000844,   1, 0x04, 0x00000000 },
-        { 0x000884,   1, 0x04, 0x00000000 },
-        { 0x0008c4,   1, 0x04, 0x00000000 },
-        { 0x000904,   1, 0x04, 0x00000000 },
-        { 0x000944,   1, 0x04, 0x00000000 },
-        { 0x000984,   1, 0x04, 0x00000000 },
-        { 0x0009c4,   1, 0x04, 0x00000000 },
-        { 0x000808,   1, 0x04, 0x00000400 },
-        { 0x000848,   1, 0x04, 0x00000400 },
-        { 0x000888,   1, 0x04, 0x00000400 },
-        { 0x0008c8,   1, 0x04, 0x00000400 },
-        { 0x000908,   1, 0x04, 0x00000400 },
-        { 0x000948,   1, 0x04, 0x00000400 },
-        { 0x000988,   1, 0x04, 0x00000400 },
-        { 0x0009c8,   1, 0x04, 0x00000400 },
-        { 0x00080c,   1, 0x04, 0x00000300 },
-        { 0x00084c,   1, 0x04, 0x00000300 },
-        { 0x00088c,   1, 0x04, 0x00000300 },
-        { 0x0008cc,   1, 0x04, 0x00000300 },
-        { 0x00090c,   1, 0x04, 0x00000300 },
-        { 0x00094c,   1, 0x04, 0x00000300 },
-        { 0x00098c,   1, 0x04, 0x00000300 },
-        { 0x0009cc,   1, 0x04, 0x00000300 },
-        { 0x000810,   1, 0x04, 0x000000cf },
-        { 0x000850,   1, 0x04, 0x00000000 },
-        { 0x000890,   1, 0x04, 0x00000000 },
-        { 0x0008d0,   1, 0x04, 0x00000000 },
-        { 0x000910,   1, 0x04, 0x00000000 },
-        { 0x000950,   1, 0x04, 0x00000000 },
-        { 0x000990,   1, 0x04, 0x00000000 },
-        { 0x0009d0,   1, 0x04, 0x00000000 },
-        { 0x000814,   1, 0x04, 0x00000040 },
-        { 0x000854,   1, 0x04, 0x00000040 },
-        { 0x000894,   1, 0x04, 0x00000040 },
-        { 0x0008d4,   1, 0x04, 0x00000040 },
-        { 0x000914,   1, 0x04, 0x00000040 },
-        { 0x000954,   1, 0x04, 0x00000040 },
-        { 0x000994,   1, 0x04, 0x00000040 },
-        { 0x0009d4,   1, 0x04, 0x00000040 },
-        { 0x000818,   1, 0x04, 0x00000001 },
-        { 0x000858,   1, 0x04, 0x00000001 },
-        { 0x000898,   1, 0x04, 0x00000001 },
-        { 0x0008d8,   1, 0x04, 0x00000001 },
-        { 0x000918,   1, 0x04, 0x00000001 },
-        { 0x000958,   1, 0x04, 0x00000001 },
-        { 0x000998,   1, 0x04, 0x00000001 },
-        { 0x0009d8,   1, 0x04, 0x00000001 },
-        { 0x00081c,   1, 0x04, 0x00000000 },
-        { 0x00085c,   1, 0x04, 0x00000000 },
-        { 0x00089c,   1, 0x04, 0x00000000 },
-        { 0x0008dc,   1, 0x04, 0x00000000 },
-        { 0x00091c,   1, 0x04, 0x00000000 },
-        { 0x00095c,   1, 0x04, 0x00000000 },
-        { 0x00099c,   1, 0x04, 0x00000000 },
-        { 0x0009dc,   1, 0x04, 0x00000000 },
-        { 0x000820,   1, 0x04, 0x00000000 },
-        { 0x000860,   1, 0x04, 0x00000000 },
-        { 0x0008a0,   1, 0x04, 0x00000000 },
-        { 0x0008e0,   1, 0x04, 0x00000000 },
-        { 0x000920,   1, 0x04, 0x00000000 },
-        { 0x000960,   1, 0x04, 0x00000000 },
-        { 0x0009a0,   1, 0x04, 0x00000000 },
-        { 0x0009e0,   1, 0x04, 0x00000000 },
-        { 0x001c00,   1, 0x04, 0x00000000 },
-        { 0x001c10,   1, 0x04, 0x00000000 },
-        { 0x001c20,   1, 0x04, 0x00000000 },
-        { 0x001c30,   1, 0x04, 0x00000000 },
-        { 0x001c40,   1, 0x04, 0x00000000 },
-        { 0x001c50,   1, 0x04, 0x00000000 },
-        { 0x001c60,   1, 0x04, 0x00000000 },
-        { 0x001c70,   1, 0x04, 0x00000000 },
-        { 0x001c80,   1, 0x04, 0x00000000 },
-        { 0x001c90,   1, 0x04, 0x00000000 },
-        { 0x001ca0,   1, 0x04, 0x00000000 },
-        { 0x001cb0,   1, 0x04, 0x00000000 },
-        { 0x001cc0,   1, 0x04, 0x00000000 },
-        { 0x001cd0,   1, 0x04, 0x00000000 },
-        { 0x001ce0,   1, 0x04, 0x00000000 },
-        { 0x001cf0,   1, 0x04, 0x00000000 },
-        { 0x001c04,   1, 0x04, 0x00000000 },
-        { 0x001c14,   1, 0x04, 0x00000000 },
-        { 0x001c24,   1, 0x04, 0x00000000 },
-        { 0x001c34,   1, 0x04, 0x00000000 },
-        { 0x001c44,   1, 0x04, 0x00000000 },
-        { 0x001c54,   1, 0x04, 0x00000000 },
-        { 0x001c64,   1, 0x04, 0x00000000 },
-        { 0x001c74,   1, 0x04, 0x00000000 },
-        { 0x001c84,   1, 0x04, 0x00000000 },
-        { 0x001c94,   1, 0x04, 0x00000000 },
-        { 0x001ca4,   1, 0x04, 0x00000000 },
-        { 0x001cb4,   1, 0x04, 0x00000000 },
-        { 0x001cc4,   1, 0x04, 0x00000000 },
-        { 0x001cd4,   1, 0x04, 0x00000000 },
-        { 0x001ce4,   1, 0x04, 0x00000000 },
-        { 0x001cf4,   1, 0x04, 0x00000000 },
-        { 0x001c08,   1, 0x04, 0x00000000 },
-        { 0x001c18,   1, 0x04, 0x00000000 },
-        { 0x001c28,   1, 0x04, 0x00000000 },
-        { 0x001c38,   1, 0x04, 0x00000000 },
-        { 0x001c48,   1, 0x04, 0x00000000 },
-        { 0x001c58,   1, 0x04, 0x00000000 },
-        { 0x001c68,   1, 0x04, 0x00000000 },
-        { 0x001c78,   1, 0x04, 0x00000000 },
-        { 0x001c88,   1, 0x04, 0x00000000 },
-        { 0x001c98,   1, 0x04, 0x00000000 },
-        { 0x001ca8,   1, 0x04, 0x00000000 },
-        { 0x001cb8,   1, 0x04, 0x00000000 },
-        { 0x001cc8,   1, 0x04, 0x00000000 },
-        { 0x001cd8,   1, 0x04, 0x00000000 },
-        { 0x001ce8,   1, 0x04, 0x00000000 },
-        { 0x001cf8,   1, 0x04, 0x00000000 },
-        { 0x001c0c,   1, 0x04, 0x00000000 },
-        { 0x001c1c,   1, 0x04, 0x00000000 },
-        { 0x001c2c,   1, 0x04, 0x00000000 },
-        { 0x001c3c,   1, 0x04, 0x00000000 },
-        { 0x001c4c,   1, 0x04, 0x00000000 },
-        { 0x001c5c,   1, 0x04, 0x00000000 },
-        { 0x001c6c,   1, 0x04, 0x00000000 },
-        { 0x001c7c,   1, 0x04, 0x00000000 },
-        { 0x001c8c,   1, 0x04, 0x00000000 },
-        { 0x001c9c,   1, 0x04, 0x00000000 },
-        { 0x001cac,   1, 0x04, 0x00000000 },
-        { 0x001cbc,   1, 0x04, 0x00000000 },
-        { 0x001ccc,   1, 0x04, 0x00000000 },
-        { 0x001cdc,   1, 0x04, 0x00000000 },
-        { 0x001cec,   1, 0x04, 0x00000000 },
-        { 0x001cfc,   2, 0x04, 0x00000000 },
-        { 0x001d10,   1, 0x04, 0x00000000 },
-        { 0x001d20,   1, 0x04, 0x00000000 },
-        { 0x001d30,   1, 0x04, 0x00000000 },
-        { 0x001d40,   1, 0x04, 0x00000000 },
-        { 0x001d50,   1, 0x04, 0x00000000 },
-        { 0x001d60,   1, 0x04, 0x00000000 },
-        { 0x001d70,   1, 0x04, 0x00000000 },
-        { 0x001d80,   1, 0x04, 0x00000000 },
-        { 0x001d90,   1, 0x04, 0x00000000 },
-        { 0x001da0,   1, 0x04, 0x00000000 },
-        { 0x001db0,   1, 0x04, 0x00000000 },
-        { 0x001dc0,   1, 0x04, 0x00000000 },
-        { 0x001dd0,   1, 0x04, 0x00000000 },
-        { 0x001de0,   1, 0x04, 0x00000000 },
-        { 0x001df0,   1, 0x04, 0x00000000 },
-        { 0x001d04,   1, 0x04, 0x00000000 },
-        { 0x001d14,   1, 0x04, 0x00000000 },
-        { 0x001d24,   1, 0x04, 0x00000000 },
-        { 0x001d34,   1, 0x04, 0x00000000 },
-        { 0x001d44,   1, 0x04, 0x00000000 },
-        { 0x001d54,   1, 0x04, 0x00000000 },
-        { 0x001d64,   1, 0x04, 0x00000000 },
-        { 0x001d74,   1, 0x04, 0x00000000 },
-        { 0x001d84,   1, 0x04, 0x00000000 },
-        { 0x001d94,   1, 0x04, 0x00000000 },
-        { 0x001da4,   1, 0x04, 0x00000000 },
-        { 0x001db4,   1, 0x04, 0x00000000 },
-        { 0x001dc4,   1, 0x04, 0x00000000 },
-        { 0x001dd4,   1, 0x04, 0x00000000 },
-        { 0x001de4,   1, 0x04, 0x00000000 },
-        { 0x001df4,   1, 0x04, 0x00000000 },
-        { 0x001d08,   1, 0x04, 0x00000000 },
-        { 0x001d18,   1, 0x04, 0x00000000 },
-        { 0x001d28,   1, 0x04, 0x00000000 },
-        { 0x001d38,   1, 0x04, 0x00000000 },
-        { 0x001d48,   1, 0x04, 0x00000000 },
-        { 0x001d58,   1, 0x04, 0x00000000 },
-        { 0x001d68,   1, 0x04, 0x00000000 },
-        { 0x001d78,   1, 0x04, 0x00000000 },
-        { 0x001d88,   1, 0x04, 0x00000000 },
-        { 0x001d98,   1, 0x04, 0x00000000 },
-        { 0x001da8,   1, 0x04, 0x00000000 },
-        { 0x001db8,   1, 0x04, 0x00000000 },
-        { 0x001dc8,   1, 0x04, 0x00000000 },
-        { 0x001dd8,   1, 0x04, 0x00000000 },
-        { 0x001de8,   1, 0x04, 0x00000000 },
-        { 0x001df8,   1, 0x04, 0x00000000 },
-        { 0x001d0c,   1, 0x04, 0x00000000 },
-        { 0x001d1c,   1, 0x04, 0x00000000 },
-        { 0x001d2c,   1, 0x04, 0x00000000 },
-        { 0x001d3c,   1, 0x04, 0x00000000 },
-        { 0x001d4c,   1, 0x04, 0x00000000 },
-        { 0x001d5c,   1, 0x04, 0x00000000 },
-        { 0x001d6c,   1, 0x04, 0x00000000 },
-        { 0x001d7c,   1, 0x04, 0x00000000 },
-        { 0x001d8c,   1, 0x04, 0x00000000 },
-        { 0x001d9c,   1, 0x04, 0x00000000 },
-        { 0x001dac,   1, 0x04, 0x00000000 },
-        { 0x001dbc,   1, 0x04, 0x00000000 },
-        { 0x001dcc,   1, 0x04, 0x00000000 },
-        { 0x001ddc,   1, 0x04, 0x00000000 },
-        { 0x001dec,   1, 0x04, 0x00000000 },
-        { 0x001dfc,   1, 0x04, 0x00000000 },
-        { 0x001f00,   1, 0x04, 0x00000000 },
-        { 0x001f08,   1, 0x04, 0x00000000 },
-        { 0x001f10,   1, 0x04, 0x00000000 },
-        { 0x001f18,   1, 0x04, 0x00000000 },
-        { 0x001f20,   1, 0x04, 0x00000000 },
-        { 0x001f28,   1, 0x04, 0x00000000 },
-        { 0x001f30,   1, 0x04, 0x00000000 },
-        { 0x001f38,   1, 0x04, 0x00000000 },
-        { 0x001f40,   1, 0x04, 0x00000000 },
-        { 0x001f48,   1, 0x04, 0x00000000 },
-        { 0x001f50,   1, 0x04, 0x00000000 },
-        { 0x001f58,   1, 0x04, 0x00000000 },
-        { 0x001f60,   1, 0x04, 0x00000000 },
-        { 0x001f68,   1, 0x04, 0x00000000 },
-        { 0x001f70,   1, 0x04, 0x00000000 },
-        { 0x001f78,   1, 0x04, 0x00000000 },
-        { 0x001f04,   1, 0x04, 0x00000000 },
-        { 0x001f0c,   1, 0x04, 0x00000000 },
-        { 0x001f14,   1, 0x04, 0x00000000 },
-        { 0x001f1c,   1, 0x04, 0x00000000 },
-        { 0x001f24,   1, 0x04, 0x00000000 },
-        { 0x001f2c,   1, 0x04, 0x00000000 },
-        { 0x001f34,   1, 0x04, 0x00000000 },
-        { 0x001f3c,   1, 0x04, 0x00000000 },
-        { 0x001f44,   1, 0x04, 0x00000000 },
-        { 0x001f4c,   1, 0x04, 0x00000000 },
-        { 0x001f54,   1, 0x04, 0x00000000 },
-        { 0x001f5c,   1, 0x04, 0x00000000 },
-        { 0x001f64,   1, 0x04, 0x00000000 },
-        { 0x001f6c,   1, 0x04, 0x00000000 },
-        { 0x001f74,   1, 0x04, 0x00000000 },
-        { 0x001f7c,   2, 0x04, 0x00000000 },
-        { 0x001f88,   1, 0x04, 0x00000000 },
-        { 0x001f90,   1, 0x04, 0x00000000 },
-        { 0x001f98,   1, 0x04, 0x00000000 },
-        { 0x001fa0,   1, 0x04, 0x00000000 },
-        { 0x001fa8,   1, 0x04, 0x00000000 },
-        { 0x001fb0,   1, 0x04, 0x00000000 },
-        { 0x001fb8,   1, 0x04, 0x00000000 },
-        { 0x001fc0,   1, 0x04, 0x00000000 },
-        { 0x001fc8,   1, 0x04, 0x00000000 },
-        { 0x001fd0,   1, 0x04, 0x00000000 },
-        { 0x001fd8,   1, 0x04, 0x00000000 },
-        { 0x001fe0,   1, 0x04, 0x00000000 },
-        { 0x001fe8,   1, 0x04, 0x00000000 },
-        { 0x001ff0,   1, 0x04, 0x00000000 },
-        { 0x001ff8,   1, 0x04, 0x00000000 },
-        { 0x001f84,   1, 0x04, 0x00000000 },
-        { 0x001f8c,   1, 0x04, 0x00000000 },
-        { 0x001f94,   1, 0x04, 0x00000000 },
-        { 0x001f9c,   1, 0x04, 0x00000000 },
-        { 0x001fa4,   1, 0x04, 0x00000000 },
-        { 0x001fac,   1, 0x04, 0x00000000 },
-        { 0x001fb4,   1, 0x04, 0x00000000 },
-        { 0x001fbc,   1, 0x04, 0x00000000 },
-        { 0x001fc4,   1, 0x04, 0x00000000 },
-        { 0x001fcc,   1, 0x04, 0x00000000 },
-        { 0x001fd4,   1, 0x04, 0x00000000 },
-        { 0x001fdc,   1, 0x04, 0x00000000 },
-        { 0x001fe4,   1, 0x04, 0x00000000 },
-        { 0x001fec,   1, 0x04, 0x00000000 },
-        { 0x001ff4,   1, 0x04, 0x00000000 },
-        { 0x001ffc,   2, 0x04, 0x00000000 },
-        { 0x002040,   1, 0x04, 0x00000011 },
-        { 0x002080,   1, 0x04, 0x00000020 },
-        { 0x0020c0,   1, 0x04, 0x00000030 },
-        { 0x002100,   1, 0x04, 0x00000040 },
-        { 0x002140,   1, 0x04, 0x00000051 },
-        { 0x00200c,   1, 0x04, 0x00000001 },
-        { 0x00204c,   1, 0x04, 0x00000001 },
-        { 0x00208c,   1, 0x04, 0x00000001 },
-        { 0x0020cc,   1, 0x04, 0x00000001 },
-        { 0x00210c,   1, 0x04, 0x00000001 },
-        { 0x00214c,   1, 0x04, 0x00000001 },
-        { 0x002010,   1, 0x04, 0x00000000 },
-        { 0x002050,   1, 0x04, 0x00000000 },
-        { 0x002090,   1, 0x04, 0x00000001 },
-        { 0x0020d0,   1, 0x04, 0x00000002 },
-        { 0x002110,   1, 0x04, 0x00000003 },
-        { 0x002150,   1, 0x04, 0x00000004 },
-        { 0x000380,   1, 0x04, 0x00000000 },
-        { 0x0003a0,   1, 0x04, 0x00000000 },
-        { 0x0003c0,   1, 0x04, 0x00000000 },
-        { 0x0003e0,   1, 0x04, 0x00000000 },
-        { 0x000384,   1, 0x04, 0x00000000 },
-        { 0x0003a4,   1, 0x04, 0x00000000 },
-        { 0x0003c4,   1, 0x04, 0x00000000 },
-        { 0x0003e4,   1, 0x04, 0x00000000 },
-        { 0x000388,   1, 0x04, 0x00000000 },
-        { 0x0003a8,   1, 0x04, 0x00000000 },
-        { 0x0003c8,   1, 0x04, 0x00000000 },
-        { 0x0003e8,   1, 0x04, 0x00000000 },
-        { 0x00038c,   1, 0x04, 0x00000000 },
-        { 0x0003ac,   1, 0x04, 0x00000000 },
-        { 0x0003cc,   1, 0x04, 0x00000000 },
-        { 0x0003ec,   1, 0x04, 0x00000000 },
-        { 0x000700,   1, 0x04, 0x00000000 },
-        { 0x000710,   1, 0x04, 0x00000000 },
-        { 0x000720,   1, 0x04, 0x00000000 },
-        { 0x000730,   1, 0x04, 0x00000000 },
-        { 0x000704,   1, 0x04, 0x00000000 },
-        { 0x000714,   1, 0x04, 0x00000000 },
-        { 0x000724,   1, 0x04, 0x00000000 },
-        { 0x000734,   1, 0x04, 0x00000000 },
-        { 0x000708,   1, 0x04, 0x00000000 },
-        { 0x000718,   1, 0x04, 0x00000000 },
-        { 0x000728,   1, 0x04, 0x00000000 },
-        { 0x000738,   1, 0x04, 0x00000000 },
-        { 0x002800, 128, 0x04, 0x00000000 },
-        { 0x000a00,   1, 0x04, 0x00000000 },
-        { 0x000a20,   1, 0x04, 0x00000000 },
-        { 0x000a40,   1, 0x04, 0x00000000 },
-        { 0x000a60,   1, 0x04, 0x00000000 },
-        { 0x000a80,   1, 0x04, 0x00000000 },
-        { 0x000aa0,   1, 0x04, 0x00000000 },
-        { 0x000ac0,   1, 0x04, 0x00000000 },
-        { 0x000ae0,   1, 0x04, 0x00000000 },
-        { 0x000b00,   1, 0x04, 0x00000000 },
-        { 0x000b20,   1, 0x04, 0x00000000 },
-        { 0x000b40,   1, 0x04, 0x00000000 },
-        { 0x000b60,   1, 0x04, 0x00000000 },
-        { 0x000b80,   1, 0x04, 0x00000000 },
-        { 0x000ba0,   1, 0x04, 0x00000000 },
-        { 0x000bc0,   1, 0x04, 0x00000000 },
-        { 0x000be0,   1, 0x04, 0x00000000 },
-        { 0x000a04,   1, 0x04, 0x00000000 },
-        { 0x000a24,   1, 0x04, 0x00000000 },
-        { 0x000a44,   1, 0x04, 0x00000000 },
-        { 0x000a64,   1, 0x04, 0x00000000 },
-        { 0x000a84,   1, 0x04, 0x00000000 },
-        { 0x000aa4,   1, 0x04, 0x00000000 },
-        { 0x000ac4,   1, 0x04, 0x00000000 },
-        { 0x000ae4,   1, 0x04, 0x00000000 },
-        { 0x000b04,   1, 0x04, 0x00000000 },
-        { 0x000b24,   1, 0x04, 0x00000000 },
-        { 0x000b44,   1, 0x04, 0x00000000 },
-        { 0x000b64,   1, 0x04, 0x00000000 },
-        { 0x000b84,   1, 0x04, 0x00000000 },
-        { 0x000ba4,   1, 0x04, 0x00000000 },
-        { 0x000bc4,   1, 0x04, 0x00000000 },
-        { 0x000be4,   1, 0x04, 0x00000000 },
-        { 0x000a08,   1, 0x04, 0x00000000 },
-        { 0x000a28,   1, 0x04, 0x00000000 },
-        { 0x000a48,   1, 0x04, 0x00000000 },
-        { 0x000a68,   1, 0x04, 0x00000000 },
-        { 0x000a88,   1, 0x04, 0x00000000 },
-        { 0x000aa8,   1, 0x04, 0x00000000 },
-        { 0x000ac8,   1, 0x04, 0x00000000 },
-        { 0x000ae8,   1, 0x04, 0x00000000 },
-        { 0x000b08,   1, 0x04, 0x00000000 },
-        { 0x000b28,   1, 0x04, 0x00000000 },
-        { 0x000b48,   1, 0x04, 0x00000000 },
-        { 0x000b68,   1, 0x04, 0x00000000 },
-        { 0x000b88,   1, 0x04, 0x00000000 },
-        { 0x000ba8,   1, 0x04, 0x00000000 },
-        { 0x000bc8,   1, 0x04, 0x00000000 },
-        { 0x000be8,   1, 0x04, 0x00000000 },
-        { 0x000a0c,   1, 0x04, 0x00000000 },
-        { 0x000a2c,   1, 0x04, 0x00000000 },
-        { 0x000a4c,   1, 0x04, 0x00000000 },
-        { 0x000a6c,   1, 0x04, 0x00000000 },
-        { 0x000a8c,   1, 0x04, 0x00000000 },
-        { 0x000aac,   1, 0x04, 0x00000000 },
-        { 0x000acc,   1, 0x04, 0x00000000 },
-        { 0x000aec,   1, 0x04, 0x00000000 },
-        { 0x000b0c,   1, 0x04, 0x00000000 },
-        { 0x000b2c,   1, 0x04, 0x00000000 },
-        { 0x000b4c,   1, 0x04, 0x00000000 },
-        { 0x000b6c,   1, 0x04, 0x00000000 },
-        { 0x000b8c,   1, 0x04, 0x00000000 },
-        { 0x000bac,   1, 0x04, 0x00000000 },
-        { 0x000bcc,   1, 0x04, 0x00000000 },
-        { 0x000bec,   1, 0x04, 0x00000000 },
-        { 0x000a10,   1, 0x04, 0x00000000 },
-        { 0x000a30,   1, 0x04, 0x00000000 },
-        { 0x000a50,   1, 0x04, 0x00000000 },
-        { 0x000a70,   1, 0x04, 0x00000000 },
-        { 0x000a90,   1, 0x04, 0x00000000 },
-        { 0x000ab0,   1, 0x04, 0x00000000 },
-        { 0x000ad0,   1, 0x04, 0x00000000 },
-        { 0x000af0,   1, 0x04, 0x00000000 },
-        { 0x000b10,   1, 0x04, 0x00000000 },
-        { 0x000b30,   1, 0x04, 0x00000000 },
-        { 0x000b50,   1, 0x04, 0x00000000 },
-        { 0x000b70,   1, 0x04, 0x00000000 },
-        { 0x000b90,   1, 0x04, 0x00000000 },
-        { 0x000bb0,   1, 0x04, 0x00000000 },
-        { 0x000bd0,   1, 0x04, 0x00000000 },
-        { 0x000bf0,   1, 0x04, 0x00000000 },
-        { 0x000a14,   1, 0x04, 0x00000000 },
-        { 0x000a34,   1, 0x04, 0x00000000 },
-        { 0x000a54,   1, 0x04, 0x00000000 },
-        { 0x000a74,   1, 0x04, 0x00000000 },
-        { 0x000a94,   1, 0x04, 0x00000000 },
-        { 0x000ab4,   1, 0x04, 0x00000000 },
-        { 0x000ad4,   1, 0x04, 0x00000000 },
-        { 0x000af4,   1, 0x04, 0x00000000 },
-        { 0x000b14,   1, 0x04, 0x00000000 },
-        { 0x000b34,   1, 0x04, 0x00000000 },
-        { 0x000b54,   1, 0x04, 0x00000000 },
-        { 0x000b74,   1, 0x04, 0x00000000 },
-        { 0x000b94,   1, 0x04, 0x00000000 },
-        { 0x000bb4,   1, 0x04, 0x00000000 },
-        { 0x000bd4,   1, 0x04, 0x00000000 },
-        { 0x000bf4,   1, 0x04, 0x00000000 },
-        { 0x000c00,   1, 0x04, 0x00000000 },
-        { 0x000c10,   1, 0x04, 0x00000000 },
-        { 0x000c20,   1, 0x04, 0x00000000 },
-        { 0x000c30,   1, 0x04, 0x00000000 },
-        { 0x000c40,   1, 0x04, 0x00000000 },
-        { 0x000c50,   1, 0x04, 0x00000000 },
-        { 0x000c60,   1, 0x04, 0x00000000 },
-        { 0x000c70,   1, 0x04, 0x00000000 },
-        { 0x000c80,   1, 0x04, 0x00000000 },
-        { 0x000c90,   1, 0x04, 0x00000000 },
-        { 0x000ca0,   1, 0x04, 0x00000000 },
-        { 0x000cb0,   1, 0x04, 0x00000000 },
-        { 0x000cc0,   1, 0x04, 0x00000000 },
-        { 0x000cd0,   1, 0x04, 0x00000000 },
-        { 0x000ce0,   1, 0x04, 0x00000000 },
-        { 0x000cf0,   1, 0x04, 0x00000000 },
-        { 0x000c04,   1, 0x04, 0x00000000 },
-        { 0x000c14,   1, 0x04, 0x00000000 },
-        { 0x000c24,   1, 0x04, 0x00000000 },
-        { 0x000c34,   1, 0x04, 0x00000000 },
-        { 0x000c44,   1, 0x04, 0x00000000 },
-        { 0x000c54,   1, 0x04, 0x00000000 },
-        { 0x000c64,   1, 0x04, 0x00000000 },
-        { 0x000c74,   1, 0x04, 0x00000000 },
-        { 0x000c84,   1, 0x04, 0x00000000 },
-        { 0x000c94,   1, 0x04, 0x00000000 },
-        { 0x000ca4,   1, 0x04, 0x00000000 },
-        { 0x000cb4,   1, 0x04, 0x00000000 },
-        { 0x000cc4,   1, 0x04, 0x00000000 },
-        { 0x000cd4,   1, 0x04, 0x00000000 },
-        { 0x000ce4,   1, 0x04, 0x00000000 },
-        { 0x000cf4,   1, 0x04, 0x00000000 },
-        { 0x000c08,   1, 0x04, 0x00000000 },
-        { 0x000c18,   1, 0x04, 0x00000000 },
-        { 0x000c28,   1, 0x04, 0x00000000 },
-        { 0x000c38,   1, 0x04, 0x00000000 },
-        { 0x000c48,   1, 0x04, 0x00000000 },
-        { 0x000c58,   1, 0x04, 0x00000000 },
-        { 0x000c68,   1, 0x04, 0x00000000 },
-        { 0x000c78,   1, 0x04, 0x00000000 },
-        { 0x000c88,   1, 0x04, 0x00000000 },
-        { 0x000c98,   1, 0x04, 0x00000000 },
-        { 0x000ca8,   1, 0x04, 0x00000000 },
-        { 0x000cb8,   1, 0x04, 0x00000000 },
-        { 0x000cc8,   1, 0x04, 0x00000000 },
-        { 0x000cd8,   1, 0x04, 0x00000000 },
-        { 0x000ce8,   1, 0x04, 0x00000000 },
-        { 0x000cf8,   1, 0x04, 0x00000000 },
-        { 0x000c0c,   1, 0x04, 0x3f800000 },
-        { 0x000c1c,   1, 0x04, 0x3f800000 },
-        { 0x000c2c,   1, 0x04, 0x3f800000 },
-        { 0x000c3c,   1, 0x04, 0x3f800000 },
-        { 0x000c4c,   1, 0x04, 0x3f800000 },
-        { 0x000c5c,   1, 0x04, 0x3f800000 },
-        { 0x000c6c,   1, 0x04, 0x3f800000 },
-        { 0x000c7c,   1, 0x04, 0x3f800000 },
-        { 0x000c8c,   1, 0x04, 0x3f800000 },
-        { 0x000c9c,   1, 0x04, 0x3f800000 },
-        { 0x000cac,   1, 0x04, 0x3f800000 },
-        { 0x000cbc,   1, 0x04, 0x3f800000 },
-        { 0x000ccc,   1, 0x04, 0x3f800000 },
-        { 0x000cdc,   1, 0x04, 0x3f800000 },
-        { 0x000cec,   1, 0x04, 0x3f800000 },
-        { 0x000cfc,   1, 0x04, 0x3f800000 },
-        { 0x000d00,   1, 0x04, 0xffff0000 },
-        { 0x000d08,   1, 0x04, 0xffff0000 },
-        { 0x000d10,   1, 0x04, 0xffff0000 },
-        { 0x000d18,   1, 0x04, 0xffff0000 },
-        { 0x000d20,   1, 0x04, 0xffff0000 },
-        { 0x000d28,   1, 0x04, 0xffff0000 },
-        { 0x000d30,   1, 0x04, 0xffff0000 },
-        { 0x000d38,   1, 0x04, 0xffff0000 },
-        { 0x000d04,   1, 0x04, 0xffff0000 },
-        { 0x000d0c,   1, 0x04, 0xffff0000 },
-        { 0x000d14,   1, 0x04, 0xffff0000 },
-        { 0x000d1c,   1, 0x04, 0xffff0000 },
-        { 0x000d24,   1, 0x04, 0xffff0000 },
-        { 0x000d2c,   1, 0x04, 0xffff0000 },
-        { 0x000d34,   1, 0x04, 0xffff0000 },
-        { 0x000d3c,   1, 0x04, 0xffff0000 },
-        { 0x000e00,   1, 0x04, 0x00000000 },
-        { 0x000e10,   1, 0x04, 0x00000000 },
-        { 0x000e20,   1, 0x04, 0x00000000 },
-        { 0x000e30,   1, 0x04, 0x00000000 },
-        { 0x000e40,   1, 0x04, 0x00000000 },
-        { 0x000e50,   1, 0x04, 0x00000000 },
-        { 0x000e60,   1, 0x04, 0x00000000 },
-        { 0x000e70,   1, 0x04, 0x00000000 },
-        { 0x000e80,   1, 0x04, 0x00000000 },
-        { 0x000e90,   1, 0x04, 0x00000000 },
-        { 0x000ea0,   1, 0x04, 0x00000000 },
-        { 0x000eb0,   1, 0x04, 0x00000000 },
-        { 0x000ec0,   1, 0x04, 0x00000000 },
-        { 0x000ed0,   1, 0x04, 0x00000000 },
-        { 0x000ee0,   1, 0x04, 0x00000000 },
-        { 0x000ef0,   1, 0x04, 0x00000000 },
-        { 0x000e04,   1, 0x04, 0xffff0000 },
-        { 0x000e14,   1, 0x04, 0xffff0000 },
-        { 0x000e24,   1, 0x04, 0xffff0000 },
-        { 0x000e34,   1, 0x04, 0xffff0000 },
-        { 0x000e44,   1, 0x04, 0xffff0000 },
-        { 0x000e54,   1, 0x04, 0xffff0000 },
-        { 0x000e64,   1, 0x04, 0xffff0000 },
-        { 0x000e74,   1, 0x04, 0xffff0000 },
-        { 0x000e84,   1, 0x04, 0xffff0000 },
-        { 0x000e94,   1, 0x04, 0xffff0000 },
-        { 0x000ea4,   1, 0x04, 0xffff0000 },
-        { 0x000eb4,   1, 0x04, 0xffff0000 },
-        { 0x000ec4,   1, 0x04, 0xffff0000 },
-        { 0x000ed4,   1, 0x04, 0xffff0000 },
-        { 0x000ee4,   1, 0x04, 0xffff0000 },
-        { 0x000ef4,   1, 0x04, 0xffff0000 },
-        { 0x000e08,   1, 0x04, 0xffff0000 },
-        { 0x000e18,   1, 0x04, 0xffff0000 },
-        { 0x000e28,   1, 0x04, 0xffff0000 },
-        { 0x000e38,   1, 0x04, 0xffff0000 },
-        { 0x000e48,   1, 0x04, 0xffff0000 },
-        { 0x000e58,   1, 0x04, 0xffff0000 },
-        { 0x000e68,   1, 0x04, 0xffff0000 },
-        { 0x000e78,   1, 0x04, 0xffff0000 },
-        { 0x000e88,   1, 0x04, 0xffff0000 },
-        { 0x000e98,   1, 0x04, 0xffff0000 },
-        { 0x000ea8,   1, 0x04, 0xffff0000 },
-        { 0x000eb8,   1, 0x04, 0xffff0000 },
-        { 0x000ec8,   1, 0x04, 0xffff0000 },
-        { 0x000ed8,   1, 0x04, 0xffff0000 },
-        { 0x000ee8,   1, 0x04, 0xffff0000 },
-        { 0x000ef8,   1, 0x04, 0xffff0000 },
-        { 0x000d40,   1, 0x04, 0x00000000 },
-        { 0x000d48,   1, 0x04, 0x00000000 },
-        { 0x000d50,   1, 0x04, 0x00000000 },
-        { 0x000d58,   1, 0x04, 0x00000000 },
-        { 0x000d44,   1, 0x04, 0x00000000 },
-        { 0x000d4c,   1, 0x04, 0x00000000 },
-        { 0x000d54,   1, 0x04, 0x00000000 },
-        { 0x000d5c,   1, 0x04, 0x00000000 },
-        { 0x001e00,   1, 0x04, 0x00000001 },
-        { 0x001e20,   1, 0x04, 0x00000001 },
-        { 0x001e40,   1, 0x04, 0x00000001 },
-        { 0x001e60,   1, 0x04, 0x00000001 },
-        { 0x001e80,   1, 0x04, 0x00000001 },
-        { 0x001ea0,   1, 0x04, 0x00000001 },
-        { 0x001ec0,   1, 0x04, 0x00000001 },
-        { 0x001ee0,   1, 0x04, 0x00000001 },
-        { 0x001e04,   1, 0x04, 0x00000001 },
-        { 0x001e24,   1, 0x04, 0x00000001 },
-        { 0x001e44,   1, 0x04, 0x00000001 },
-        { 0x001e64,   1, 0x04, 0x00000001 },
-        { 0x001e84,   1, 0x04, 0x00000001 },
-        { 0x001ea4,   1, 0x04, 0x00000001 },
-        { 0x001ec4,   1, 0x04, 0x00000001 },
-        { 0x001ee4,   1, 0x04, 0x00000001 },
-        { 0x001e08,   1, 0x04, 0x00000002 },
-        { 0x001e28,   1, 0x04, 0x00000002 },
-        { 0x001e48,   1, 0x04, 0x00000002 },
-        { 0x001e68,   1, 0x04, 0x00000002 },
-        { 0x001e88,   1, 0x04, 0x00000002 },
-        { 0x001ea8,   1, 0x04, 0x00000002 },
-        { 0x001ec8,   1, 0x04, 0x00000002 },
-        { 0x001ee8,   1, 0x04, 0x00000002 },
-        { 0x001e0c,   1, 0x04, 0x00000001 },
-        { 0x001e2c,   1, 0x04, 0x00000001 },
-        { 0x001e4c,   1, 0x04, 0x00000001 },
-        { 0x001e6c,   1, 0x04, 0x00000001 },
-        { 0x001e8c,   1, 0x04, 0x00000001 },
-        { 0x001eac,   1, 0x04, 0x00000001 },
-        { 0x001ecc,   1, 0x04, 0x00000001 },
-        { 0x001eec,   1, 0x04, 0x00000001 },
-        { 0x001e10,   1, 0x04, 0x00000001 },
-        { 0x001e30,   1, 0x04, 0x00000001 },
-        { 0x001e50,   1, 0x04, 0x00000001 },
-        { 0x001e70,   1, 0x04, 0x00000001 },
-        { 0x001e90,   1, 0x04, 0x00000001 },
-        { 0x001eb0,   1, 0x04, 0x00000001 },
-        { 0x001ed0,   1, 0x04, 0x00000001 },
-        { 0x001ef0,   1, 0x04, 0x00000001 },
-        { 0x001e14,   1, 0x04, 0x00000002 },
-        { 0x001e34,   1, 0x04, 0x00000002 },
-        { 0x001e54,   1, 0x04, 0x00000002 },
-        { 0x001e74,   1, 0x04, 0x00000002 },
-        { 0x001e94,   1, 0x04, 0x00000002 },
-        { 0x001eb4,   1, 0x04, 0x00000002 },
-        { 0x001ed4,   1, 0x04, 0x00000002 },
-        { 0x001ef4,   1, 0x04, 0x00000002 },
-        { 0x001e18,   1, 0x04, 0x00000001 },
-        { 0x001e38,   1, 0x04, 0x00000001 },
-        { 0x001e58,   1, 0x04, 0x00000001 },
-        { 0x001e78,   1, 0x04, 0x00000001 },
-        { 0x001e98,   1, 0x04, 0x00000001 },
-        { 0x001eb8,   1, 0x04, 0x00000001 },
-        { 0x001ed8,   1, 0x04, 0x00000001 },
-        { 0x001ef8,   1, 0x04, 0x00000001 },
-        { 0x003400, 128, 0x04, 0x00000000 },
-        { 0x00030c,   1, 0x04, 0x00000001 },
-        { 0x001944,   1, 0x04, 0x00000000 },
-        { 0x001514,   1, 0x04, 0x00000000 },
-        { 0x000d68,   1, 0x04, 0x0000ffff },
-        { 0x00121c,   1, 0x04, 0x0fac6881 },
-        { 0x000fac,   1, 0x04, 0x00000001 },
-        { 0x001538,   1, 0x04, 0x00000001 },
-        { 0x000fe0,   2, 0x04, 0x00000000 },
-        { 0x000fe8,   1, 0x04, 0x00000014 },
-        { 0x000fec,   1, 0x04, 0x00000040 },
-        { 0x000ff0,   1, 0x04, 0x00000000 },
-        { 0x00179c,   1, 0x04, 0x00000000 },
-        { 0x001228,   1, 0x04, 0x00000400 },
-        { 0x00122c,   1, 0x04, 0x00000300 },
-        { 0x001230,   1, 0x04, 0x00010001 },
-        { 0x0007f8,   1, 0x04, 0x00000000 },
-        { 0x0015b4,   1, 0x04, 0x00000001 },
-        { 0x0015cc,   1, 0x04, 0x00000000 },
-        { 0x001534,   1, 0x04, 0x00000000 },
-        { 0x000fb0,   1, 0x04, 0x00000000 },
-        { 0x0015d0,   1, 0x04, 0x00000000 },
-        { 0x00153c,   1, 0x04, 0x00000000 },
-        { 0x0016b4,   1, 0x04, 0x00000003 },
-        { 0x000fbc,   4, 0x04, 0x0000ffff },
-        { 0x000df8,   2, 0x04, 0x00000000 },
-        { 0x001948,   1, 0x04, 0x00000000 },
-        { 0x001970,   1, 0x04, 0x00000001 },
-        { 0x00161c,   1, 0x04, 0x000009f0 },
-        { 0x000dcc,   1, 0x04, 0x00000010 },
-        { 0x00163c,   1, 0x04, 0x00000000 },
-        { 0x0015e4,   1, 0x04, 0x00000000 },
-        { 0x001160,  32, 0x04, 0x25e00040 },
-        { 0x001880,  32, 0x04, 0x00000000 },
-        { 0x000f84,   2, 0x04, 0x00000000 },
-        { 0x0017c8,   2, 0x04, 0x00000000 },
-        { 0x0017d0,   1, 0x04, 0x000000ff },
-        { 0x0017d4,   1, 0x04, 0xffffffff },
-        { 0x0017d8,   1, 0x04, 0x00000002 },
-        { 0x0017dc,   1, 0x04, 0x00000000 },
-        { 0x0015f4,   2, 0x04, 0x00000000 },
-        { 0x001434,   2, 0x04, 0x00000000 },
-        { 0x000d74,   1, 0x04, 0x00000000 },
-        { 0x000dec,   1, 0x04, 0x00000001 },
-        { 0x0013a4,   1, 0x04, 0x00000000 },
-        { 0x001318,   1, 0x04, 0x00000001 },
-        { 0x001644,   1, 0x04, 0x00000000 },
-        { 0x000748,   1, 0x04, 0x00000000 },
-        { 0x000de8,   1, 0x04, 0x00000000 },
-        { 0x001648,   1, 0x04, 0x00000000 },
-        { 0x0012a4,   1, 0x04, 0x00000000 },
-        { 0x001120,   4, 0x04, 0x00000000 },
-        { 0x001118,   1, 0x04, 0x00000000 },
-        { 0x00164c,   1, 0x04, 0x00000000 },
-        { 0x001658,   1, 0x04, 0x00000000 },
-        { 0x001910,   1, 0x04, 0x00000290 },
-        { 0x001518,   1, 0x04, 0x00000000 },
-        { 0x00165c,   1, 0x04, 0x00000001 },
-        { 0x001520,   1, 0x04, 0x00000000 },
-        { 0x001604,   1, 0x04, 0x00000000 },
-        { 0x001570,   1, 0x04, 0x00000000 },
-        { 0x0013b0,   2, 0x04, 0x3f800000 },
-        { 0x00020c,   1, 0x04, 0x00000000 },
-        { 0x001670,   1, 0x04, 0x30201000 },
-        { 0x001674,   1, 0x04, 0x70605040 },
-        { 0x001678,   1, 0x04, 0xb8a89888 },
-        { 0x00167c,   1, 0x04, 0xf8e8d8c8 },
-        { 0x00166c,   1, 0x04, 0x00000000 },
-        { 0x001680,   1, 0x04, 0x00ffff00 },
-        { 0x0012d0,   1, 0x04, 0x00000003 },
-        { 0x0012d4,   1, 0x04, 0x00000002 },
-        { 0x001684,   2, 0x04, 0x00000000 },
-        { 0x000dac,   2, 0x04, 0x00001b02 },
-        { 0x000db4,   1, 0x04, 0x00000000 },
-        { 0x00168c,   1, 0x04, 0x00000000 },
-        { 0x0015bc,   1, 0x04, 0x00000000 },
-        { 0x00156c,   1, 0x04, 0x00000000 },
-        { 0x00187c,   1, 0x04, 0x00000000 },
-        { 0x001110,   1, 0x04, 0x00000001 },
-        { 0x000dc0,   3, 0x04, 0x00000000 },
-        { 0x001234,   1, 0x04, 0x00000000 },
-        { 0x001690,   1, 0x04, 0x00000000 },
-        { 0x0012ac,   1, 0x04, 0x00000001 },
-        { 0x0002c4,   1, 0x04, 0x00000000 },
-        { 0x000790,   5, 0x04, 0x00000000 },
-        { 0x00077c,   1, 0x04, 0x00000000 },
-        { 0x001000,   1, 0x04, 0x00000010 },
-        { 0x0010fc,   1, 0x04, 0x00000000 },
-        { 0x001290,   1, 0x04, 0x00000000 },
-        { 0x000218,   1, 0x04, 0x00000010 },
-        { 0x0012d8,   1, 0x04, 0x00000000 },
-        { 0x0012dc,   1, 0x04, 0x00000010 },
-        { 0x000d94,   1, 0x04, 0x00000001 },
-        { 0x00155c,   2, 0x04, 0x00000000 },
-        { 0x001564,   1, 0x04, 0x00000fff },
-        { 0x001574,   2, 0x04, 0x00000000 },
-        { 0x00157c,   1, 0x04, 0x000fffff },
-        { 0x001354,   1, 0x04, 0x00000000 },
-        { 0x001610,   1, 0x04, 0x00000012 },
-        { 0x001608,   2, 0x04, 0x00000000 },
-        { 0x00260c,   1, 0x04, 0x00000000 },
-        { 0x0007ac,   1, 0x04, 0x00000000 },
-        { 0x00162c,   1, 0x04, 0x00000003 },
-        { 0x000210,   1, 0x04, 0x00000000 },
-        { 0x000320,   1, 0x04, 0x00000000 },
-        { 0x000324,   6, 0x04, 0x3f800000 },
-        { 0x000750,   1, 0x04, 0x00000000 },
-        { 0x000760,   1, 0x04, 0x39291909 },
-        { 0x000764,   1, 0x04, 0x79695949 },
-        { 0x000768,   1, 0x04, 0xb9a99989 },
-        { 0x00076c,   1, 0x04, 0xf9e9d9c9 },
-        { 0x000770,   1, 0x04, 0x30201000 },
-        { 0x000774,   1, 0x04, 0x70605040 },
-        { 0x000778,   1, 0x04, 0x00009080 },
-        { 0x000780,   1, 0x04, 0x39291909 },
-        { 0x000784,   1, 0x04, 0x79695949 },
-        { 0x000788,   1, 0x04, 0xb9a99989 },
-        { 0x00078c,   1, 0x04, 0xf9e9d9c9 },
-        { 0x0007d0,   1, 0x04, 0x30201000 },
-        { 0x0007d4,   1, 0x04, 0x70605040 },
-        { 0x0007d8,   1, 0x04, 0x00009080 },
-        { 0x00037c,   1, 0x04, 0x00000001 },
-        { 0x000740,   2, 0x04, 0x00000000 },
-        { 0x002600,   1, 0x04, 0x00000000 },
-        { 0x001918,   1, 0x04, 0x00000000 },
-        { 0x00191c,   1, 0x04, 0x00000900 },
-        { 0x001920,   1, 0x04, 0x00000405 },
-        { 0x001308,   1, 0x04, 0x00000001 },
-        { 0x001924,   1, 0x04, 0x00000000 },
-        { 0x0013ac,   1, 0x04, 0x00000000 },
-        { 0x00192c,   1, 0x04, 0x00000001 },
-        { 0x00193c,   1, 0x04, 0x00002c1c },
-        { 0x000d7c,   1, 0x04, 0x00000000 },
-        { 0x000f8c,   1, 0x04, 0x00000000 },
-        { 0x0002c0,   1, 0x04, 0x00000001 },
-        { 0x001510,   1, 0x04, 0x00000000 },
-        { 0x001940,   1, 0x04, 0x00000000 },
-        { 0x000ff4,   2, 0x04, 0x00000000 },
-        { 0x00194c,   2, 0x04, 0x00000000 },
-        { 0x001968,   1, 0x04, 0x00000000 },
-        { 0x001590,   1, 0x04, 0x0000003f },
-        { 0x0007e8,   4, 0x04, 0x00000000 },
-        { 0x00196c,   1, 0x04, 0x00000011 },
-        { 0x0002e4,   1, 0x04, 0x0000b001 },
-        { 0x00036c,   2, 0x04, 0x00000000 },
-        { 0x00197c,   1, 0x04, 0x00000000 },
-        { 0x000fcc,   2, 0x04, 0x00000000 },
-        { 0x0002d8,   1, 0x04, 0x00000040 },
-        { 0x001980,   1, 0x04, 0x00000080 },
-        { 0x001504,   1, 0x04, 0x00000080 },
-        { 0x001984,   1, 0x04, 0x00000000 },
-        { 0x000300,   1, 0x04, 0x00000001 },
-        { 0x0013a8,   1, 0x04, 0x00000000 },
-        { 0x0012ec,   1, 0x04, 0x00000000 },
-        { 0x001310,   1, 0x04, 0x00000000 },
-        { 0x001314,   1, 0x04, 0x00000001 },
-        { 0x001380,   1, 0x04, 0x00000000 },
-        { 0x001384,   4, 0x04, 0x00000001 },
-        { 0x001394,   1, 0x04, 0x00000000 },
-        { 0x00139c,   1, 0x04, 0x00000000 },
-        { 0x001398,   1, 0x04, 0x00000000 },
-        { 0x001594,   1, 0x04, 0x00000000 },
-        { 0x001598,   4, 0x04, 0x00000001 },
-        { 0x000f54,   3, 0x04, 0x00000000 },
-        { 0x0019bc,   1, 0x04, 0x00000000 },
-        { 0x000f9c,   2, 0x04, 0x00000000 },
-        { 0x0012cc,   1, 0x04, 0x00000000 },
-        { 0x0012e8,   1, 0x04, 0x00000000 },
-        { 0x00130c,   1, 0x04, 0x00000001 },
-        { 0x001360,   8, 0x04, 0x00000000 },
-        { 0x00133c,   2, 0x04, 0x00000001 },
-        { 0x001344,   1, 0x04, 0x00000002 },
-        { 0x001348,   2, 0x04, 0x00000001 },
-        { 0x001350,   1, 0x04, 0x00000002 },
-        { 0x001358,   1, 0x04, 0x00000001 },
-        { 0x0012e4,   1, 0x04, 0x00000000 },
-        { 0x00131c,   4, 0x04, 0x00000000 },
-        { 0x0019c0,   1, 0x04, 0x00000000 },
-        { 0x001140,   1, 0x04, 0x00000000 },
-        { 0x0019c4,   1, 0x04, 0x00000000 },
-        { 0x0019c8,   1, 0x04, 0x00001500 },
-        { 0x00135c,   1, 0x04, 0x00000000 },
-        { 0x000f90,   1, 0x04, 0x00000000 },
-        { 0x0019e0,   8, 0x04, 0x00000001 },
-        { 0x0019cc,   1, 0x04, 0x00000001 },
-        { 0x0015b8,   1, 0x04, 0x00000000 },
-        { 0x001a00,   1, 0x04, 0x00001111 },
-        { 0x001a04,   7, 0x04, 0x00000000 },
-        { 0x000d6c,   2, 0x04, 0xffff0000 },
-        { 0x0010f8,   1, 0x04, 0x00001010 },
-        { 0x000d80,   5, 0x04, 0x00000000 },
-        { 0x000da0,   1, 0x04, 0x00000000 },
-        { 0x0007a4,   2, 0x04, 0x00000000 },
-        { 0x001508,   1, 0x04, 0x80000000 },
-        { 0x00150c,   1, 0x04, 0x40000000 },
-        { 0x001668,   1, 0x04, 0x00000000 },
-        { 0x000318,   2, 0x04, 0x00000008 },
-        { 0x000d9c,   1, 0x04, 0x00000001 },
-        { 0x000ddc,   1, 0x04, 0x00000002 },
-        { 0x000374,   1, 0x04, 0x00000000 },
-        { 0x000378,   1, 0x04, 0x00000020 },
-        { 0x0007dc,   1, 0x04, 0x00000000 },
-        { 0x00074c,   1, 0x04, 0x00000055 },
-        { 0x001420,   1, 0x04, 0x00000003 },
-        { 0x0017bc,   2, 0x04, 0x00000000 },
-        { 0x0017c4,   1, 0x04, 0x00000001 },
-        { 0x001008,   1, 0x04, 0x00000008 },
-        { 0x00100c,   1, 0x04, 0x00000040 },
-        { 0x001010,   1, 0x04, 0x0000012c },
-        { 0x000d60,   1, 0x04, 0x00000040 },
-        { 0x00075c,   1, 0x04, 0x00000003 },
-        { 0x001018,   1, 0x04, 0x00000020 },
-        { 0x00101c,   1, 0x04, 0x00000001 },
-        { 0x001020,   1, 0x04, 0x00000020 },
-        { 0x001024,   1, 0x04, 0x00000001 },
-        { 0x001444,   3, 0x04, 0x00000000 },
-        { 0x000360,   1, 0x04, 0x20164010 },
-        { 0x000364,   1, 0x04, 0x00000020 },
-        { 0x000368,   1, 0x04, 0x00000000 },
-        { 0x000de4,   1, 0x04, 0x00000000 },
-        { 0x000204,   1, 0x04, 0x00000006 },
-        { 0x000208,   1, 0x04, 0x00000000 },
-        { 0x0002cc,   2, 0x04, 0x003fffff },
-        { 0x001220,   1, 0x04, 0x00000005 },
-        { 0x000fdc,   1, 0x04, 0x00000000 },
-        { 0x000f98,   1, 0x04, 0x00400008 },
-        { 0x001284,   1, 0x04, 0x08000080 },
-        { 0x001450,   1, 0x04, 0x00400008 },
-        { 0x001454,   1, 0x04, 0x08000080 },
-        { 0x000214,   1, 0x04, 0x00000000 },
+static const struct nvc0_graph_pack
+nv108_grctx_pack_icmd[] = {
+        { nv108_grctx_init_icmd_0 },
         {}
 };
 
-static struct nvc0_graph_init
-nv108_grctx_init_unk40xx[] = {
+static const struct nvc0_graph_init
+nv108_grctx_init_fe_0[] = {
         { 0x404004,   8, 0x04, 0x00000000 },
         { 0x404024,   1, 0x04, 0x0000e000 },
         { 0x404028,   8, 0x04, 0x00000000 },
@@ -1132,8 +311,8 @@ nv108_grctx_init_unk40xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nv108_grctx_init_unk58xx[] = {
+static const struct nvc0_graph_init
+nv108_grctx_init_ds_0[] = {
         { 0x405800,   1, 0x04, 0x0f8000bf },
         { 0x405830,   1, 0x04, 0x02180648 },
         { 0x405834,   1, 0x04, 0x08000000 },
@@ -1146,8 +325,10 @@ nv108_grctx_init_unk58xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nv108_grctx_init_unk64xx[] = {
+static const struct nvc0_graph_init
+nv108_grctx_init_pd_0[] = {
+        { 0x406020,   1, 0x04, 0x034103c1 },
+        { 0x406028,   4, 0x04, 0x00000001 },
         { 0x4064a8,   1, 0x04, 0x00000000 },
         { 0x4064ac,   1, 0x04, 0x00003fff },
         { 0x4064b0,   3, 0x04, 0x00000000 },
@@ -1159,8 +340,8 @@ nv108_grctx_init_unk64xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nv108_grctx_init_unk78xx[] = {
+const struct nvc0_graph_init
+nv108_grctx_init_rstr2d_0[] = {
         { 0x407804,   1, 0x04, 0x00000063 },
         { 0x40780c,   1, 0x04, 0x0a418820 },
         { 0x407810,   1, 0x04, 0x062080e6 },
@@ -1172,8 +353,8 @@ nv108_grctx_init_unk78xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nv108_grctx_init_unk88xx[] = {
+static const struct nvc0_graph_init
+nv108_grctx_init_be_0[] = {
         { 0x408800,   1, 0x04, 0x32802a3c },
         { 0x408804,   1, 0x04, 0x00000040 },
         { 0x408808,   1, 0x04, 0x1003e005 },
@@ -1185,9 +366,23 @@ nv108_grctx_init_unk88xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nv108_grctx_init_gpc_0[] = {
-        { 0x418380,   1, 0x04, 0x00000016 },
+static const struct nvc0_graph_pack
+nv108_grctx_pack_hub[] = {
+        { nvc0_grctx_init_main_0 },
+        { nv108_grctx_init_fe_0 },
+        { nvf0_grctx_init_pri_0 },
+        { nve4_grctx_init_memfmt_0 },
+        { nv108_grctx_init_ds_0 },
+        { nvf0_grctx_init_cwd_0 },
+        { nv108_grctx_init_pd_0 },
+        { nv108_grctx_init_rstr2d_0 },
+        { nve4_grctx_init_scc_0 },
+        { nv108_grctx_init_be_0 },
+        {}
+};
+
+const struct nvc0_graph_init
+nv108_grctx_init_prop_0[] = {
         { 0x418400,   1, 0x04, 0x38005e00 },
         { 0x418404,   1, 0x04, 0x71e0ffff },
         { 0x41840c,   1, 0x04, 0x00001008 },
@@ -1196,11 +391,21 @@ nv108_grctx_init_gpc_0[] = {
         { 0x418450,   6, 0x04, 0x00000000 },
         { 0x418468,   1, 0x04, 0x00000001 },
         { 0x41846c,   2, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nv108_grctx_init_gpc_unk_1[] = {
         { 0x418600,   1, 0x04, 0x0000007f },
         { 0x418684,   1, 0x04, 0x0000001f },
         { 0x418700,   1, 0x04, 0x00000002 },
         { 0x418704,   2, 0x04, 0x00000080 },
         { 0x41870c,   2, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nv108_grctx_init_setup_0[] = {
         { 0x418800,   1, 0x04, 0x7006863a },
         { 0x418808,   1, 0x04, 0x00000000 },
         { 0x41880c,   1, 0x04, 0x00000030 },
@@ -1211,10 +416,11 @@ nv108_grctx_init_gpc_0[] = {
         { 0x4188e0,   1, 0x04, 0x01000000 },
         { 0x4188e8,   5, 0x04, 0x00000000 },
         { 0x4188fc,   1, 0x04, 0x20100058 },
-        { 0x41891c,   1, 0x04, 0x00ff00ff },
-        { 0x418924,   1, 0x04, 0x00000000 },
-        { 0x418928,   1, 0x04, 0x00ffff00 },
-        { 0x41892c,   1, 0x04, 0x0000ff00 },
+        {}
+};
+
+const struct nvc0_graph_init
+nv108_grctx_init_crstr_0[] = {
         { 0x418b00,   1, 0x04, 0x0000001e },
         { 0x418b08,   1, 0x04, 0x0a418820 },
         { 0x418b0c,   1, 0x04, 0x062080e6 },
@@ -1223,24 +429,36 @@ nv108_grctx_init_gpc_0[] = {
         { 0x418b18,   1, 0x04, 0x0a418820 },
         { 0x418b1c,   1, 0x04, 0x000000e6 },
         { 0x418bb8,   1, 0x04, 0x00000103 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nv108_grctx_init_gpm_0[] = {
         { 0x418c08,   1, 0x04, 0x00000001 },
         { 0x418c10,   8, 0x04, 0x00000000 },
         { 0x418c40,   1, 0x04, 0xffffffff },
         { 0x418c6c,   1, 0x04, 0x00000001 },
         { 0x418c80,   1, 0x04, 0x2020000c },
         { 0x418c8c,   1, 0x04, 0x00000001 },
-        { 0x418d24,   1, 0x04, 0x00000000 },
-        { 0x419000,   1, 0x04, 0x00000780 },
-        { 0x419004,   2, 0x04, 0x00000000 },
-        { 0x419014,   1, 0x04, 0x00000004 },
         {}
 };
 
-static struct nvc0_graph_init
-nv108_grctx_init_tpc[] = {
-        { 0x419848,   1, 0x04, 0x00000000 },
-        { 0x419864,   1, 0x04, 0x00000129 },
-        { 0x419888,   1, 0x04, 0x00000000 },
+static const struct nvc0_graph_pack
+nv108_grctx_pack_gpc[] = {
+        { nvc0_grctx_init_gpc_unk_0 },
+        { nv108_grctx_init_prop_0 },
+        { nv108_grctx_init_gpc_unk_1 },
+        { nv108_grctx_init_setup_0 },
+        { nvc0_grctx_init_zcull_0 },
+        { nv108_grctx_init_crstr_0 },
+        { nv108_grctx_init_gpm_0 },
+        { nvf0_grctx_init_gpc_unk_2 },
+        { nvc0_grctx_init_gcc_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nv108_grctx_init_tex_0[] = {
         { 0x419a00,   1, 0x04, 0x000100f0 },
         { 0x419a04,   1, 0x04, 0x00000001 },
         { 0x419a08,   1, 0x04, 0x00000421 },
@@ -1251,14 +469,11 @@ nv108_grctx_init_tpc[] = {
         { 0x419a20,   1, 0x04, 0x00000800 },
         { 0x419a30,   1, 0x04, 0x00000001 },
         { 0x419ac4,   1, 0x04, 0x0037f440 },
-        { 0x419c00,   1, 0x04, 0x0000001a },
-        { 0x419c04,   1, 0x04, 0x80000006 },
-        { 0x419c08,   1, 0x04, 0x00000002 },
-        { 0x419c20,   1, 0x04, 0x00000000 },
-        { 0x419c24,   1, 0x04, 0x00084210 },
-        { 0x419c28,   1, 0x04, 0x3efbefbe },
-        { 0x419ce8,   1, 0x04, 0x00000000 },
-        { 0x419cf4,   1, 0x04, 0x00000203 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nv108_grctx_init_sm_0[] = {
         { 0x419e04,   1, 0x04, 0x00000000 },
         { 0x419e08,   1, 0x04, 0x0000001d },
         { 0x419e0c,   1, 0x04, 0x00000000 },
@@ -1272,7 +487,7 @@ nv108_grctx_init_tpc[] = {
         { 0x419e68,   1, 0x04, 0x00000002 },
         { 0x419e6c,  12, 0x04, 0x00000000 },
         { 0x419eac,   1, 0x04, 0x00001f8f },
-        { 0x419eb0,   1, 0x04, 0x0db00da0 },
+        { 0x419eb0,   1, 0x04, 0x0db00d2f },
         { 0x419eb8,   1, 0x04, 0x00000000 },
         { 0x419ec8,   1, 0x04, 0x0001304f },
         { 0x419f30,   4, 0x04, 0x00000000 },
@@ -1285,25 +500,37 @@ nv108_grctx_init_tpc[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nv108_grctx_init_unk[] = {
-        { 0x41be24,   1, 0x04, 0x00000006 },
+static const struct nvc0_graph_pack
+nv108_grctx_pack_tpc[] = {
+        { nvd7_grctx_init_pe_0 },
+        { nv108_grctx_init_tex_0 },
+        { nvf0_grctx_init_mpc_0 },
+        { nvf0_grctx_init_l1c_0 },
+        { nv108_grctx_init_sm_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nv108_grctx_init_cbm_0[] = {
         { 0x41bec0,   1, 0x04, 0x10000000 },
         { 0x41bec4,   1, 0x04, 0x00037f7f },
         { 0x41bee4,   1, 0x04, 0x00000000 },
         { 0x41bef0,   1, 0x04, 0x000003ff },
-        { 0x41bf00,   1, 0x04, 0x0a418820 },
-        { 0x41bf04,   1, 0x04, 0x062080e6 },
-        { 0x41bf08,   1, 0x04, 0x020398a4 },
-        { 0x41bf0c,   1, 0x04, 0x0e629062 },
-        { 0x41bf10,   1, 0x04, 0x0a418820 },
-        { 0x41bf14,   1, 0x04, 0x000000e6 },
-        { 0x41bfd0,   1, 0x04, 0x00900103 },
-        { 0x41bfe0,   1, 0x04, 0x00400001 },
-        { 0x41bfe4,   1, 0x04, 0x00000000 },
         {}
 };
 
+static const struct nvc0_graph_pack
+nv108_grctx_pack_ppc[] = {
+        { nve4_grctx_init_pes_0 },
+        { nv108_grctx_init_cbm_0 },
+        { nvd7_grctx_init_wwdx_0 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH context implementation
+ ******************************************************************************/
+
 static void
 nv108_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
 {
@@ -1346,47 +573,6 @@ nv108_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
         mmio_list(0x17e920, 0x00090d08, 0, 0);
 }
 
-static struct nvc0_graph_init *
-nv108_grctx_init_hub[] = {
-        nvc0_grctx_init_base,
-        nv108_grctx_init_unk40xx,
-        nvf0_grctx_init_unk44xx,
-        nve4_grctx_init_unk46xx,
-        nve4_grctx_init_unk47xx,
-        nv108_grctx_init_unk58xx,
-        nvf0_grctx_init_unk5bxx,
-        nvf0_grctx_init_unk60xx,
-        nv108_grctx_init_unk64xx,
-        nv108_grctx_init_unk78xx,
-        nve4_grctx_init_unk80xx,
-        nv108_grctx_init_unk88xx,
-        NULL
-};
-
-struct nvc0_graph_init *
-nv108_grctx_init_gpc[] = {
-        nv108_grctx_init_gpc_0,
-        nvc0_grctx_init_gpc_1,
-        nv108_grctx_init_tpc,
-        nv108_grctx_init_unk,
-        NULL
-};
-
-struct nvc0_graph_init
-nv108_grctx_init_mthd_magic[] = {
-        { 0x3410, 1, 0x04, 0x8e0e2006 },
-        { 0x3414, 1, 0x04, 0x00000038 },
-        {}
-};
-
-static struct nvc0_graph_mthd
-nv108_grctx_init_mthd[] = {
-        { 0xa197, nv108_grctx_init_a197, },
-        { 0x902d, nvc0_grctx_init_902d, },
-        { 0x902d, nv108_grctx_init_mthd_magic, },
-        {}
-};
-
 struct nouveau_oclass *
 nv108_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .base.handle = NV_ENGCTX(GR, 0x08),
@@ -1398,11 +584,14 @@ nv108_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .rd32 = _nouveau_graph_context_rd32,
                 .wr32 = _nouveau_graph_context_wr32,
         },
-        .main = nve4_grctx_generate_main,
-        .mods = nv108_grctx_generate_mods,
-        .unkn = nve4_grctx_generate_unkn,
-        .hub  = nv108_grctx_init_hub,
-        .gpc  = nv108_grctx_init_gpc,
-        .icmd = nv108_grctx_init_icmd,
-        .mthd = nv108_grctx_init_mthd,
+        .main  = nve4_grctx_generate_main,
+        .mods  = nv108_grctx_generate_mods,
+        .unkn  = nve4_grctx_generate_unkn,
+        .hub   = nv108_grctx_pack_hub,
+        .gpc   = nv108_grctx_pack_gpc,
+        .zcull = nvc0_grctx_pack_zcull,
+        .tpc   = nv108_grctx_pack_tpc,
+        .ppc   = nv108_grctx_pack_ppc,
+        .icmd  = nv108_grctx_pack_icmd,
+        .mthd  = nvf0_grctx_pack_mthd,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.c
index fe67415c3e17..833a96508c4e 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.c
@@ -22,10 +22,14 @@
  * Authors: Ben Skeggs
  */
 
-#include "nvc0.h"
+#include "ctxnvc0.h"
 
-struct nvc0_graph_init
-nvc0_grctx_init_icmd[] = {
+/*******************************************************************************
+ * PGRAPH context register lists
+ ******************************************************************************/
+
+static const struct nvc0_graph_init
+nvc0_grctx_init_icmd_0[] = {
         { 0x001000,   1, 0x01, 0x00000004 },
         { 0x0000a9,   1, 0x01, 0x0000ffff },
         { 0x000038,   1, 0x01, 0x0fac6881 },
@@ -140,8 +144,7 @@ nvc0_grctx_init_icmd[] = {
         { 0x000586,   1, 0x01, 0x00000040 },
         { 0x000582,   2, 0x01, 0x00000080 },
         { 0x0005c2,   1, 0x01, 0x00000001 },
-        { 0x000638,   1, 0x01, 0x00000001 },
-        { 0x000639,   1, 0x01, 0x00000001 },
+        { 0x000638,   2, 0x01, 0x00000001 },
         { 0x00063a,   1, 0x01, 0x00000002 },
         { 0x00063b,   2, 0x01, 0x00000001 },
         { 0x00063d,   1, 0x01, 0x00000002 },
@@ -201,15 +204,13 @@ nvc0_grctx_init_icmd[] = {
         { 0x000787,   1, 0x01, 0x000000cf },
         { 0x00078c,   1, 0x01, 0x00000008 },
         { 0x000792,   1, 0x01, 0x00000001 },
-        { 0x000794,   1, 0x01, 0x00000001 },
-        { 0x000795,   2, 0x01, 0x00000001 },
+        { 0x000794,   3, 0x01, 0x00000001 },
         { 0x000797,   1, 0x01, 0x000000cf },
         { 0x000836,   1, 0x01, 0x00000001 },
         { 0x00079a,   1, 0x01, 0x00000002 },
         { 0x000833,   1, 0x01, 0x04444480 },
         { 0x0007a1,   1, 0x01, 0x00000001 },
-        { 0x0007a3,   1, 0x01, 0x00000001 },
-        { 0x0007a4,   2, 0x01, 0x00000001 },
+        { 0x0007a3,   3, 0x01, 0x00000001 },
         { 0x000831,   1, 0x01, 0x00000004 },
         { 0x00080c,   1, 0x01, 0x00000002 },
         { 0x00080d,   2, 0x01, 0x00000100 },
@@ -235,14 +236,12 @@ nvc0_grctx_init_icmd[] = {
         { 0x0006b1,   1, 0x01, 0x00000011 },
         { 0x00078c,   1, 0x01, 0x00000008 },
         { 0x000792,   1, 0x01, 0x00000001 },
-        { 0x000794,   1, 0x01, 0x00000001 },
-        { 0x000795,   2, 0x01, 0x00000001 },
+        { 0x000794,   3, 0x01, 0x00000001 },
         { 0x000797,   1, 0x01, 0x000000cf },
         { 0x00079a,   1, 0x01, 0x00000002 },
         { 0x000833,   1, 0x01, 0x04444480 },
         { 0x0007a1,   1, 0x01, 0x00000001 },
-        { 0x0007a3,   1, 0x01, 0x00000001 },
-        { 0x0007a4,   2, 0x01, 0x00000001 },
+        { 0x0007a3,   3, 0x01, 0x00000001 },
         { 0x000831,   1, 0x01, 0x00000004 },
         { 0x01e100,   1, 0x01, 0x00000001 },
         { 0x001000,   1, 0x01, 0x00000014 },
@@ -267,8 +266,14 @@ nvc0_grctx_init_icmd[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_9097[] = {
+const struct nvc0_graph_pack
+nvc0_grctx_pack_icmd[] = {
+        { nvc0_grctx_init_icmd_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvc0_grctx_init_9097_0[] = {
         { 0x000800,   8, 0x40, 0x00000000 },
         { 0x000804,   8, 0x40, 0x00000000 },
         { 0x000808,   8, 0x40, 0x00000400 },
@@ -516,8 +521,7 @@ nvc0_grctx_init_9097[] = {
         { 0x001350,   1, 0x04, 0x00000002 },
         { 0x001358,   1, 0x04, 0x00000001 },
         { 0x0012e4,   1, 0x04, 0x00000000 },
-        { 0x00131c,   1, 0x04, 0x00000000 },
-        { 0x001320,   3, 0x04, 0x00000000 },
+        { 0x00131c,   4, 0x04, 0x00000000 },
         { 0x0019c0,   1, 0x04, 0x00000000 },
         { 0x001140,   1, 0x04, 0x00000000 },
         { 0x0019c4,   1, 0x04, 0x00000000 },
@@ -571,8 +575,8 @@ nvc0_grctx_init_9097[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_902d[] = {
+const struct nvc0_graph_init
+nvc0_grctx_init_902d_0[] = {
         { 0x000200,   1, 0x04, 0x000000cf },
         { 0x000204,   1, 0x04, 0x00000001 },
         { 0x000208,   1, 0x04, 0x00000020 },
@@ -590,8 +594,8 @@ nvc0_grctx_init_902d[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_9039[] = {
+const struct nvc0_graph_init
+nvc0_grctx_init_9039_0[] = {
         { 0x00030c,   3, 0x04, 0x00000000 },
         { 0x000320,   1, 0x04, 0x00000000 },
         { 0x000238,   2, 0x04, 0x00000000 },
@@ -599,8 +603,8 @@ nvc0_grctx_init_9039[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_90c0[] = {
+const struct nvc0_graph_init
+nvc0_grctx_init_90c0_0[] = {
         { 0x00270c,   8, 0x20, 0x00000000 },
         { 0x00030c,   1, 0x04, 0x00000001 },
         { 0x001944,   1, 0x04, 0x00000000 },
@@ -617,38 +621,44 @@ nvc0_grctx_init_90c0[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_base[] = {
+const struct nvc0_graph_pack
+nvc0_grctx_pack_mthd[] = {
+        { nvc0_grctx_init_9097_0, 0x9097 },
+        { nvc0_grctx_init_902d_0, 0x902d },
+        { nvc0_grctx_init_9039_0, 0x9039 },
+        { nvc0_grctx_init_90c0_0, 0x90c0 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_grctx_init_main_0[] = {
         { 0x400204,   2, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_unk40xx[] = {
-        { 0x404004,  10, 0x04, 0x00000000 },
+const struct nvc0_graph_init
+nvc0_grctx_init_fe_0[] = {
+        { 0x404004,  11, 0x04, 0x00000000 },
         { 0x404044,   1, 0x04, 0x00000000 },
-        { 0x404094,   1, 0x04, 0x00000000 },
-        { 0x404098,  12, 0x04, 0x00000000 },
+        { 0x404094,  13, 0x04, 0x00000000 },
         { 0x4040c8,   1, 0x04, 0xf0000087 },
         { 0x4040d0,   6, 0x04, 0x00000000 },
         { 0x4040e8,   1, 0x04, 0x00001000 },
         { 0x4040f8,   1, 0x04, 0x00000000 },
-        { 0x404130,   1, 0x04, 0x00000000 },
-        { 0x404134,   1, 0x04, 0x00000000 },
+        { 0x404130,   2, 0x04, 0x00000000 },
         { 0x404138,   1, 0x04, 0x20000040 },
         { 0x404150,   1, 0x04, 0x0000002e },
         { 0x404154,   1, 0x04, 0x00000400 },
         { 0x404158,   1, 0x04, 0x00000200 },
         { 0x404164,   1, 0x04, 0x00000055 },
         { 0x404168,   1, 0x04, 0x00000000 },
-        { 0x404174,   1, 0x04, 0x00000000 },
-        { 0x404178,   2, 0x04, 0x00000000 },
+        { 0x404174,   3, 0x04, 0x00000000 },
         { 0x404200,   8, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_unk44xx[] = {
+const struct nvc0_graph_init
+nvc0_grctx_init_pri_0[] = {
         { 0x404404,  14, 0x04, 0x00000000 },
         { 0x404460,   2, 0x04, 0x00000000 },
         { 0x404468,   1, 0x04, 0x00ffffff },
@@ -658,8 +668,8 @@ nvc0_grctx_init_unk44xx[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_unk46xx[] = {
+const struct nvc0_graph_init
+nvc0_grctx_init_memfmt_0[] = {
         { 0x404604,   1, 0x04, 0x00000015 },
         { 0x404608,   1, 0x04, 0x00000000 },
         { 0x40460c,   1, 0x04, 0x00002e00 },
@@ -674,19 +684,14 @@ nvc0_grctx_init_unk46xx[] = {
         { 0x4046a0,   1, 0x04, 0x007f0080 },
         { 0x4046a4,  18, 0x04, 0x00000000 },
         { 0x4046f0,   2, 0x04, 0x00000000 },
-        {}
-};
-
-struct nvc0_graph_init
-nvc0_grctx_init_unk47xx[] = {
         { 0x404700,  13, 0x04, 0x00000000 },
         { 0x404734,   1, 0x04, 0x00000100 },
         { 0x404738,   8, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_unk58xx[] = {
+static const struct nvc0_graph_init
+nvc0_grctx_init_ds_0[] = {
         { 0x405800,   1, 0x04, 0x078000bf },
         { 0x405830,   1, 0x04, 0x02180000 },
         { 0x405834,   2, 0x04, 0x00000000 },
@@ -697,23 +702,18 @@ nvc0_grctx_init_unk58xx[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_unk60xx[] = {
+static const struct nvc0_graph_init
+nvc0_grctx_init_pd_0[] = {
         { 0x406020,   1, 0x04, 0x000103c1 },
         { 0x406028,   4, 0x04, 0x00000001 },
-        {}
-};
-
-struct nvc0_graph_init
-nvc0_grctx_init_unk64xx[] = {
         { 0x4064a8,   1, 0x04, 0x00000000 },
         { 0x4064ac,   1, 0x04, 0x00003fff },
         { 0x4064b4,   2, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_unk78xx[] = {
+const struct nvc0_graph_init
+nvc0_grctx_init_rstr2d_0[] = {
         { 0x407804,   1, 0x04, 0x00000023 },
         { 0x40780c,   1, 0x04, 0x0a418820 },
         { 0x407810,   1, 0x04, 0x062080e6 },
@@ -725,8 +725,8 @@ nvc0_grctx_init_unk78xx[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_unk80xx[] = {
+const struct nvc0_graph_init
+nvc0_grctx_init_scc_0[] = {
         { 0x408000,   2, 0x04, 0x00000000 },
         { 0x408008,   1, 0x04, 0x00000018 },
         { 0x40800c,   2, 0x04, 0x00000000 },
@@ -736,8 +736,8 @@ nvc0_grctx_init_unk80xx[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_rop[] = {
+static const struct nvc0_graph_init
+nvc0_grctx_init_be_0[] = {
         { 0x408800,   1, 0x04, 0x02802a3c },
         { 0x408804,   1, 0x04, 0x00000040 },
         { 0x408808,   1, 0x04, 0x0003e00d },
@@ -748,9 +748,28 @@ nvc0_grctx_init_rop[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_gpc_0[] = {
+const struct nvc0_graph_pack
+nvc0_grctx_pack_hub[] = {
+        { nvc0_grctx_init_main_0 },
+        { nvc0_grctx_init_fe_0 },
+        { nvc0_grctx_init_pri_0 },
+        { nvc0_grctx_init_memfmt_0 },
+        { nvc0_grctx_init_ds_0 },
+        { nvc0_grctx_init_pd_0 },
+        { nvc0_grctx_init_rstr2d_0 },
+        { nvc0_grctx_init_scc_0 },
+        { nvc0_grctx_init_be_0 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_grctx_init_gpc_unk_0[] = {
         { 0x418380,   1, 0x04, 0x00000016 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_grctx_init_prop_0[] = {
         { 0x418400,   1, 0x04, 0x38004e00 },
         { 0x418404,   1, 0x04, 0x71e0ffff },
         { 0x418408,   1, 0x04, 0x00000000 },
@@ -760,6 +779,11 @@ nvc0_grctx_init_gpc_0[] = {
         { 0x418450,   6, 0x04, 0x00000000 },
         { 0x418468,   1, 0x04, 0x00000001 },
         { 0x41846c,   2, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_grctx_init_gpc_unk_1[] = {
         { 0x418600,   1, 0x04, 0x0000001f },
         { 0x418684,   1, 0x04, 0x0000000f },
         { 0x418700,   1, 0x04, 0x00000002 },
@@ -767,6 +791,11 @@ nvc0_grctx_init_gpc_0[] = {
         { 0x418708,   1, 0x04, 0x00000000 },
         { 0x41870c,   1, 0x04, 0x07c80000 },
         { 0x418710,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvc0_grctx_init_setup_0[] = {
         { 0x418800,   1, 0x04, 0x0006860a },
         { 0x418808,   3, 0x04, 0x00000000 },
         { 0x418828,   1, 0x04, 0x00008442 },
@@ -775,10 +804,20 @@ nvc0_grctx_init_gpc_0[] = {
         { 0x4188e0,   1, 0x04, 0x01000000 },
         { 0x4188e8,   5, 0x04, 0x00000000 },
         { 0x4188fc,   1, 0x04, 0x00100000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_grctx_init_zcull_0[] = {
         { 0x41891c,   1, 0x04, 0x00ff00ff },
         { 0x418924,   1, 0x04, 0x00000000 },
         { 0x418928,   1, 0x04, 0x00ffff00 },
         { 0x41892c,   1, 0x04, 0x0000ff00 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_grctx_init_crstr_0[] = {
         { 0x418b00,   1, 0x04, 0x00000000 },
         { 0x418b08,   1, 0x04, 0x0a418820 },
         { 0x418b0c,   1, 0x04, 0x062080e6 },
@@ -787,18 +826,41 @@ nvc0_grctx_init_gpc_0[] = {
         { 0x418b18,   1, 0x04, 0x0a418820 },
         { 0x418b1c,   1, 0x04, 0x000000e6 },
         { 0x418bb8,   1, 0x04, 0x00000103 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_grctx_init_gpm_0[] = {
         { 0x418c08,   1, 0x04, 0x00000001 },
         { 0x418c10,   8, 0x04, 0x00000000 },
         { 0x418c80,   1, 0x04, 0x20200004 },
         { 0x418c8c,   1, 0x04, 0x00000001 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_grctx_init_gcc_0[] = {
         { 0x419000,   1, 0x04, 0x00000780 },
         { 0x419004,   2, 0x04, 0x00000000 },
         { 0x419014,   1, 0x04, 0x00000004 },
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_gpc_1[] = {
+const struct nvc0_graph_pack
+nvc0_grctx_pack_gpc[] = {
+        { nvc0_grctx_init_gpc_unk_0 },
+        { nvc0_grctx_init_prop_0 },
+        { nvc0_grctx_init_gpc_unk_1 },
+        { nvc0_grctx_init_setup_0 },
+        { nvc0_grctx_init_zcull_0 },
+        { nvc0_grctx_init_crstr_0 },
+        { nvc0_grctx_init_gpm_0 },
+        { nvc0_grctx_init_gcc_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvc0_grctx_init_zcullr_0[] = {
         { 0x418a00,   3, 0x04, 0x00000000 },
         { 0x418a0c,   1, 0x04, 0x00010000 },
         { 0x418a10,   3, 0x04, 0x00000000 },
@@ -826,19 +888,35 @@ nvc0_grctx_init_gpc_1[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc0_grctx_init_tpc[] = {
+const struct nvc0_graph_pack
+nvc0_grctx_pack_zcull[] = {
+        { nvc0_grctx_init_zcullr_0 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_grctx_init_pe_0[] = {
         { 0x419818,   1, 0x04, 0x00000000 },
         { 0x41983c,   1, 0x04, 0x00038bc7 },
         { 0x419848,   1, 0x04, 0x00000000 },
         { 0x419864,   1, 0x04, 0x0000012a },
         { 0x419888,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvc0_grctx_init_tex_0[] = {
         { 0x419a00,   1, 0x04, 0x000001f0 },
         { 0x419a04,   1, 0x04, 0x00000001 },
         { 0x419a08,   1, 0x04, 0x00000023 },
         { 0x419a0c,   1, 0x04, 0x00020000 },
         { 0x419a10,   1, 0x04, 0x00000000 },
         { 0x419a14,   1, 0x04, 0x00000200 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_grctx_init_wwdx_0[] = {
         { 0x419b00,   1, 0x04, 0x0a418820 },
         { 0x419b04,   1, 0x04, 0x062080e6 },
         { 0x419b08,   1, 0x04, 0x020398a4 },
@@ -848,15 +926,35 @@ nvc0_grctx_init_tpc[] = {
         { 0x419bd0,   1, 0x04, 0x00900103 },
         { 0x419be0,   1, 0x04, 0x00000001 },
         { 0x419be4,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_grctx_init_mpc_0[] = {
         { 0x419c00,   1, 0x04, 0x00000002 },
         { 0x419c04,   1, 0x04, 0x00000006 },
         { 0x419c08,   1, 0x04, 0x00000002 },
         { 0x419c20,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvc0_grctx_init_l1c_0[] = {
         { 0x419cb0,   1, 0x04, 0x00060048 },
         { 0x419ce8,   1, 0x04, 0x00000000 },
         { 0x419cf4,   1, 0x04, 0x00000183 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_grctx_init_tpccs_0[] = {
         { 0x419d20,   1, 0x04, 0x02180000 },
         { 0x419d24,   1, 0x04, 0x00001fff },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvc0_grctx_init_sm_0[] = {
         { 0x419e04,   3, 0x04, 0x00000000 },
         { 0x419e10,   1, 0x04, 0x00000002 },
         { 0x419e44,   1, 0x04, 0x001beff2 },
@@ -868,6 +966,22 @@ nvc0_grctx_init_tpc[] = {
         {}
 };
 
+const struct nvc0_graph_pack
+nvc0_grctx_pack_tpc[] = {
+        { nvc0_grctx_init_pe_0 },
+        { nvc0_grctx_init_tex_0 },
+        { nvc0_grctx_init_wwdx_0 },
+        { nvc0_grctx_init_mpc_0 },
+        { nvc0_grctx_init_l1c_0 },
+        { nvc0_grctx_init_tpccs_0 },
+        { nvc0_grctx_init_sm_0 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH context implementation
+ ******************************************************************************/
+
 void
 nvc0_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
 {
@@ -1055,14 +1169,14 @@ void
 nvc0_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
 {
         struct nvc0_grctx_oclass *oclass = (void *)nv_engine(priv)->cclass;
-        int i;
 
         nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
 
-        for (i = 0; oclass->hub[i]; i++)
-                nvc0_graph_mmio(priv, oclass->hub[i]);
-        for (i = 0; oclass->gpc[i]; i++)
-                nvc0_graph_mmio(priv, oclass->gpc[i]);
+        nvc0_graph_mmio(priv, oclass->hub);
+        nvc0_graph_mmio(priv, oclass->gpc);
+        nvc0_graph_mmio(priv, oclass->zcull);
+        nvc0_graph_mmio(priv, oclass->tpc);
+        nvc0_graph_mmio(priv, oclass->ppc);
 
         nv_wr32(priv, 0x404154, 0x00000000);
 
@@ -1182,46 +1296,6 @@ done:
         return ret;
 }
 
-struct nvc0_graph_init *
-nvc0_grctx_init_hub[] = {
-        nvc0_grctx_init_base,
-        nvc0_grctx_init_unk40xx,
-        nvc0_grctx_init_unk44xx,
-        nvc0_grctx_init_unk46xx,
-        nvc0_grctx_init_unk47xx,
-        nvc0_grctx_init_unk58xx,
-        nvc0_grctx_init_unk60xx,
-        nvc0_grctx_init_unk64xx,
-        nvc0_grctx_init_unk78xx,
-        nvc0_grctx_init_unk80xx,
-        nvc0_grctx_init_rop,
-        NULL
-};
-
-static struct nvc0_graph_init *
-nvc0_grctx_init_gpc[] = {
-        nvc0_grctx_init_gpc_0,
-        nvc0_grctx_init_gpc_1,
-        nvc0_grctx_init_tpc,
-        NULL
-};
-
-struct nvc0_graph_init
-nvc0_grctx_init_mthd_magic[] = {
-        { 0x3410, 1, 0x04, 0x00000000 },
-        {}
-};
-
-struct nvc0_graph_mthd
-nvc0_grctx_init_mthd[] = {
-        { 0x9097, nvc0_grctx_init_9097, },
-        { 0x902d, nvc0_grctx_init_902d, },
-        { 0x9039, nvc0_grctx_init_9039, },
-        { 0x90c0, nvc0_grctx_init_90c0, },
-        { 0x902d, nvc0_grctx_init_mthd_magic, },
-        {}
-};
-
 struct nouveau_oclass *
 nvc0_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .base.handle = NV_ENGCTX(GR, 0xc0),
@@ -1233,11 +1307,13 @@ nvc0_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .rd32 = _nouveau_graph_context_rd32,
                 .wr32 = _nouveau_graph_context_wr32,
         },
-        .main = nvc0_grctx_generate_main,
-        .mods = nvc0_grctx_generate_mods,
-        .unkn = nvc0_grctx_generate_unkn,
-        .hub  = nvc0_grctx_init_hub,
-        .gpc  = nvc0_grctx_init_gpc,
-        .icmd = nvc0_grctx_init_icmd,
-        .mthd = nvc0_grctx_init_mthd,
+        .main  = nvc0_grctx_generate_main,
+        .mods  = nvc0_grctx_generate_mods,
+        .unkn  = nvc0_grctx_generate_unkn,
+        .hub   = nvc0_grctx_pack_hub,
+        .gpc   = nvc0_grctx_pack_gpc,
+        .zcull = nvc0_grctx_pack_zcull,
+        .tpc   = nvc0_grctx_pack_tpc,
+        .icmd  = nvc0_grctx_pack_icmd,
+        .mthd  = nvc0_grctx_pack_mthd,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.h b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.h
new file mode 100644
index 000000000000..9c815d1f99ef
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc0.h
@@ -0,0 +1,170 @@
+#ifndef __NVKM_GRCTX_NVC0_H__
+#define __NVKM_GRCTX_NVC0_H__
+
+#include "nvc0.h"
+
+struct nvc0_grctx {
+        struct nvc0_graph_priv *priv;
+        struct nvc0_graph_data *data;
+        struct nvc0_graph_mmio *mmio;
+        int buffer_nr;
+        u64 buffer[4];
+        u64 addr;
+};
+
+struct nvc0_grctx_oclass {
+        struct nouveau_oclass base;
+        /* main context generation function */
+        void  (*main)(struct nvc0_graph_priv *, struct nvc0_grctx *);
+        /* context-specific modify-on-first-load list generation function */
+        void  (*mods)(struct nvc0_graph_priv *, struct nvc0_grctx *);
+        void  (*unkn)(struct nvc0_graph_priv *);
+        /* mmio context data */
+        const struct nvc0_graph_pack *hub;
+        const struct nvc0_graph_pack *gpc;
+        const struct nvc0_graph_pack *zcull;
+        const struct nvc0_graph_pack *tpc;
+        const struct nvc0_graph_pack *ppc;
+        /* indirect context data, generated with icmds/mthds */
+        const struct nvc0_graph_pack *icmd;
+        const struct nvc0_graph_pack *mthd;
+};
+
+#define mmio_data(s,a,p) do {                                                  \
+        info->buffer[info->buffer_nr] = round_up(info->addr, (a));             \
+        info->addr = info->buffer[info->buffer_nr++] + (s);                    \
+        info->data->size = (s);                                                \
+        info->data->align = (a);                                               \
+        info->data->access = (p);                                              \
+        info->data++;                                                          \
+} while(0)
+
+#define mmio_list(r,d,s,b) do {                                                \
+        info->mmio->addr = (r);                                                \
+        info->mmio->data = (d);                                                \
+        info->mmio->shift = (s);                                               \
+        info->mmio->buffer = (b);                                              \
+        info->mmio++;                                                          \
+        nv_wr32(priv, (r), (d) | ((s) ? (info->buffer[(b)] >> (s)) : 0));      \
+} while(0)
+
+extern struct nouveau_oclass *nvc0_grctx_oclass;
+int  nvc0_grctx_generate(struct nvc0_graph_priv *);
+void nvc0_grctx_generate_main(struct nvc0_graph_priv *, struct nvc0_grctx *);
+void nvc0_grctx_generate_mods(struct nvc0_graph_priv *, struct nvc0_grctx *);
+void nvc0_grctx_generate_unkn(struct nvc0_graph_priv *);
+void nvc0_grctx_generate_tpcid(struct nvc0_graph_priv *);
+void nvc0_grctx_generate_r406028(struct nvc0_graph_priv *);
+void nvc0_grctx_generate_r4060a8(struct nvc0_graph_priv *);
+void nvc0_grctx_generate_r418bb8(struct nvc0_graph_priv *);
+void nvc0_grctx_generate_r406800(struct nvc0_graph_priv *);
+
+extern struct nouveau_oclass *nvc1_grctx_oclass;
+void nvc1_grctx_generate_mods(struct nvc0_graph_priv *, struct nvc0_grctx *);
+void nvc1_grctx_generate_unkn(struct nvc0_graph_priv *);
+
+extern struct nouveau_oclass *nvc4_grctx_oclass;
+extern struct nouveau_oclass *nvc8_grctx_oclass;
+extern struct nouveau_oclass *nvd7_grctx_oclass;
+extern struct nouveau_oclass *nvd9_grctx_oclass;
+
+extern struct nouveau_oclass *nve4_grctx_oclass;
+void nve4_grctx_generate_main(struct nvc0_graph_priv *, struct nvc0_grctx *);
+void nve4_grctx_generate_unkn(struct nvc0_graph_priv *);
+void nve4_grctx_generate_r418bb8(struct nvc0_graph_priv *);
+
+extern struct nouveau_oclass *nvf0_grctx_oclass;
+extern struct nouveau_oclass *nv108_grctx_oclass;
+extern struct nouveau_oclass *gm107_grctx_oclass;
+
+/* context init value lists */
+
+extern const struct nvc0_graph_pack nvc0_grctx_pack_icmd[];
+
+extern const struct nvc0_graph_pack nvc0_grctx_pack_mthd[];
+extern const struct nvc0_graph_init nvc0_grctx_init_902d_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_9039_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_90c0_0[];
+
+extern const struct nvc0_graph_pack nvc0_grctx_pack_hub[];
+extern const struct nvc0_graph_init nvc0_grctx_init_main_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_fe_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_pri_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_memfmt_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_rstr2d_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_scc_0[];
+
+extern const struct nvc0_graph_pack nvc0_grctx_pack_gpc[];
+extern const struct nvc0_graph_init nvc0_grctx_init_gpc_unk_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_prop_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_gpc_unk_1[];
+extern const struct nvc0_graph_init nvc0_grctx_init_zcull_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_crstr_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_gpm_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_gcc_0[];
+
+extern const struct nvc0_graph_pack nvc0_grctx_pack_zcull[];
+
+extern const struct nvc0_graph_pack nvc0_grctx_pack_tpc[];
+extern const struct nvc0_graph_init nvc0_grctx_init_pe_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_wwdx_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_mpc_0[];
+extern const struct nvc0_graph_init nvc0_grctx_init_tpccs_0[];
+
+extern const struct nvc0_graph_init nvc4_grctx_init_tex_0[];
+extern const struct nvc0_graph_init nvc4_grctx_init_l1c_0[];
+extern const struct nvc0_graph_init nvc4_grctx_init_sm_0[];
+
+extern const struct nvc0_graph_init nvc1_grctx_init_9097_0[];
+
+extern const struct nvc0_graph_init nvc1_grctx_init_gpm_0[];
+
+extern const struct nvc0_graph_init nvc1_grctx_init_pe_0[];
+extern const struct nvc0_graph_init nvc1_grctx_init_wwdx_0[];
+extern const struct nvc0_graph_init nvc1_grctx_init_tpccs_0[];
+
+extern const struct nvc0_graph_init nvc8_grctx_init_9197_0[];
+extern const struct nvc0_graph_init nvc8_grctx_init_9297_0[];
+
+extern const struct nvc0_graph_pack nvd9_grctx_pack_icmd[];
+
+extern const struct nvc0_graph_pack nvd9_grctx_pack_mthd[];
+
+extern const struct nvc0_graph_init nvd9_grctx_init_fe_0[];
+extern const struct nvc0_graph_init nvd9_grctx_init_be_0[];
+
+extern const struct nvc0_graph_init nvd9_grctx_init_prop_0[];
+extern const struct nvc0_graph_init nvd9_grctx_init_gpc_unk_1[];
+extern const struct nvc0_graph_init nvd9_grctx_init_crstr_0[];
+
+extern const struct nvc0_graph_init nvd9_grctx_init_sm_0[];
+
+extern const struct nvc0_graph_init nvd7_grctx_init_pe_0[];
+
+extern const struct nvc0_graph_init nvd7_grctx_init_wwdx_0[];
+
+extern const struct nvc0_graph_init nve4_grctx_init_memfmt_0[];
+extern const struct nvc0_graph_init nve4_grctx_init_ds_0[];
+extern const struct nvc0_graph_init nve4_grctx_init_scc_0[];
+
+extern const struct nvc0_graph_init nve4_grctx_init_gpm_0[];
+
+extern const struct nvc0_graph_init nve4_grctx_init_pes_0[];
+
+extern const struct nvc0_graph_pack nvf0_grctx_pack_mthd[];
+
+extern const struct nvc0_graph_init nvf0_grctx_init_pri_0[];
+extern const struct nvc0_graph_init nvf0_grctx_init_cwd_0[];
+
+extern const struct nvc0_graph_init nvf0_grctx_init_gpc_unk_2[];
+
+extern const struct nvc0_graph_init nvf0_grctx_init_mpc_0[];
+extern const struct nvc0_graph_init nvf0_grctx_init_l1c_0[];
+
+extern const struct nvc0_graph_init nv108_grctx_init_rstr2d_0[];
+
+extern const struct nvc0_graph_init nv108_grctx_init_prop_0[];
+extern const struct nvc0_graph_init nv108_grctx_init_crstr_0[];
+
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc1.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc1.c
index 71b4283f7fad..24a92c569c0a 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc1.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc1.c
@@ -22,10 +22,14 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
-#include "nvc0.h"
+#include "ctxnvc0.h"
 
-static struct nvc0_graph_init
-nvc1_grctx_init_icmd[] = {
+/*******************************************************************************
+ * PGRAPH context register lists
+ ******************************************************************************/
+
+static const struct nvc0_graph_init
+nvc1_grctx_init_icmd_0[] = {
         { 0x001000,   1, 0x01, 0x00000004 },
         { 0x0000a9,   1, 0x01, 0x0000ffff },
         { 0x000038,   1, 0x01, 0x0fac6881 },
@@ -141,8 +145,7 @@ nvc1_grctx_init_icmd[] = {
         { 0x000586,   1, 0x01, 0x00000040 },
         { 0x000582,   2, 0x01, 0x00000080 },
         { 0x0005c2,   1, 0x01, 0x00000001 },
-        { 0x000638,   1, 0x01, 0x00000001 },
-        { 0x000639,   1, 0x01, 0x00000001 },
+        { 0x000638,   2, 0x01, 0x00000001 },
         { 0x00063a,   1, 0x01, 0x00000002 },
         { 0x00063b,   2, 0x01, 0x00000001 },
         { 0x00063d,   1, 0x01, 0x00000002 },
@@ -202,15 +205,13 @@ nvc1_grctx_init_icmd[] = {
         { 0x000787,   1, 0x01, 0x000000cf },
         { 0x00078c,   1, 0x01, 0x00000008 },
         { 0x000792,   1, 0x01, 0x00000001 },
-        { 0x000794,   1, 0x01, 0x00000001 },
-        { 0x000795,   2, 0x01, 0x00000001 },
+        { 0x000794,   3, 0x01, 0x00000001 },
         { 0x000797,   1, 0x01, 0x000000cf },
         { 0x000836,   1, 0x01, 0x00000001 },
         { 0x00079a,   1, 0x01, 0x00000002 },
         { 0x000833,   1, 0x01, 0x04444480 },
         { 0x0007a1,   1, 0x01, 0x00000001 },
-        { 0x0007a3,   1, 0x01, 0x00000001 },
-        { 0x0007a4,   2, 0x01, 0x00000001 },
+        { 0x0007a3,   3, 0x01, 0x00000001 },
         { 0x000831,   1, 0x01, 0x00000004 },
         { 0x00080c,   1, 0x01, 0x00000002 },
         { 0x00080d,   2, 0x01, 0x00000100 },
@@ -236,14 +237,12 @@ nvc1_grctx_init_icmd[] = {
         { 0x0006b1,   1, 0x01, 0x00000011 },
         { 0x00078c,   1, 0x01, 0x00000008 },
         { 0x000792,   1, 0x01, 0x00000001 },
-        { 0x000794,   1, 0x01, 0x00000001 },
-        { 0x000795,   2, 0x01, 0x00000001 },
+        { 0x000794,   3, 0x01, 0x00000001 },
         { 0x000797,   1, 0x01, 0x000000cf },
         { 0x00079a,   1, 0x01, 0x00000002 },
         { 0x000833,   1, 0x01, 0x04444480 },
         { 0x0007a1,   1, 0x01, 0x00000001 },
-        { 0x0007a3,   1, 0x01, 0x00000001 },
-        { 0x0007a4,   2, 0x01, 0x00000001 },
+        { 0x0007a3,   3, 0x01, 0x00000001 },
         { 0x000831,   1, 0x01, 0x00000004 },
         { 0x01e100,   1, 0x01, 0x00000001 },
         { 0x001000,   1, 0x01, 0x00000014 },
@@ -268,8 +267,14 @@ nvc1_grctx_init_icmd[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc1_grctx_init_9097[] = {
+static const struct nvc0_graph_pack
+nvc1_grctx_pack_icmd[] = {
+        { nvc1_grctx_init_icmd_0 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc1_grctx_init_9097_0[] = {
         { 0x000800,   8, 0x40, 0x00000000 },
         { 0x000804,   8, 0x40, 0x00000000 },
         { 0x000808,   8, 0x40, 0x00000400 },
@@ -516,8 +521,7 @@ nvc1_grctx_init_9097[] = {
         { 0x001350,   1, 0x04, 0x00000002 },
         { 0x001358,   1, 0x04, 0x00000001 },
         { 0x0012e4,   1, 0x04, 0x00000000 },
-        { 0x00131c,   1, 0x04, 0x00000000 },
-        { 0x001320,   3, 0x04, 0x00000000 },
+        { 0x00131c,   4, 0x04, 0x00000000 },
         { 0x0019c0,   1, 0x04, 0x00000000 },
         { 0x001140,   1, 0x04, 0x00000000 },
         { 0x0019c4,   1, 0x04, 0x00000000 },
@@ -571,15 +575,25 @@ nvc1_grctx_init_9097[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nvc1_grctx_init_9197[] = {
+static const struct nvc0_graph_init
+nvc1_grctx_init_9197_0[] = {
         { 0x003400, 128, 0x04, 0x00000000 },
         { 0x0002e4,   1, 0x04, 0x0000b001 },
         {}
 };
 
-static struct nvc0_graph_init
-nvc1_grctx_init_unk58xx[] = {
+static const struct nvc0_graph_pack
+nvc1_grctx_pack_mthd[] = {
+        { nvc1_grctx_init_9097_0, 0x9097 },
+        { nvc1_grctx_init_9197_0, 0x9197 },
+        { nvc0_grctx_init_902d_0, 0x902d },
+        { nvc0_grctx_init_9039_0, 0x9039 },
+        { nvc0_grctx_init_90c0_0, 0x90c0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvc1_grctx_init_ds_0[] = {
         { 0x405800,   1, 0x04, 0x0f8000bf },
         { 0x405830,   1, 0x04, 0x02180218 },
         { 0x405834,   2, 0x04, 0x00000000 },
@@ -590,8 +604,20 @@ nvc1_grctx_init_unk58xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nvc1_grctx_init_rop[] = {
+static const struct nvc0_graph_init
+nvc1_grctx_init_pd_0[] = {
+        { 0x406020,   1, 0x04, 0x000103c1 },
+        { 0x406028,   4, 0x04, 0x00000001 },
+        { 0x4064a8,   1, 0x04, 0x00000000 },
+        { 0x4064ac,   1, 0x04, 0x00003fff },
+        { 0x4064b4,   2, 0x04, 0x00000000 },
+        { 0x4064c0,   1, 0x04, 0x80140078 },
+        { 0x4064c4,   1, 0x04, 0x0086ffff },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvc1_grctx_init_be_0[] = {
         { 0x408800,   1, 0x04, 0x02802a3c },
         { 0x408804,   1, 0x04, 0x00000040 },
         { 0x408808,   1, 0x04, 0x1003e005 },
@@ -602,25 +628,22 @@ nvc1_grctx_init_rop[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nvc1_grctx_init_gpc_0[] = {
-        { 0x418380,   1, 0x04, 0x00000016 },
-        { 0x418400,   1, 0x04, 0x38004e00 },
-        { 0x418404,   1, 0x04, 0x71e0ffff },
-        { 0x418408,   1, 0x04, 0x00000000 },
-        { 0x41840c,   1, 0x04, 0x00001008 },
-        { 0x418410,   1, 0x04, 0x0fff0fff },
-        { 0x418414,   1, 0x04, 0x00200fff },
-        { 0x418450,   6, 0x04, 0x00000000 },
-        { 0x418468,   1, 0x04, 0x00000001 },
-        { 0x41846c,   2, 0x04, 0x00000000 },
-        { 0x418600,   1, 0x04, 0x0000001f },
-        { 0x418684,   1, 0x04, 0x0000000f },
-        { 0x418700,   1, 0x04, 0x00000002 },
-        { 0x418704,   1, 0x04, 0x00000080 },
-        { 0x418708,   1, 0x04, 0x00000000 },
-        { 0x41870c,   1, 0x04, 0x07c80000 },
-        { 0x418710,   1, 0x04, 0x00000000 },
+static const struct nvc0_graph_pack
+nvc1_grctx_pack_hub[] = {
+        { nvc0_grctx_init_main_0 },
+        { nvc0_grctx_init_fe_0 },
+        { nvc0_grctx_init_pri_0 },
+        { nvc0_grctx_init_memfmt_0 },
+        { nvc1_grctx_init_ds_0 },
+        { nvc1_grctx_init_pd_0 },
+        { nvc0_grctx_init_rstr2d_0 },
+        { nvc0_grctx_init_scc_0 },
+        { nvc1_grctx_init_be_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvc1_grctx_init_setup_0[] = {
         { 0x418800,   1, 0x04, 0x0006860a },
         { 0x418808,   3, 0x04, 0x00000000 },
         { 0x418828,   1, 0x04, 0x00008442 },
@@ -629,69 +652,44 @@ nvc1_grctx_init_gpc_0[] = {
         { 0x4188e0,   1, 0x04, 0x01000000 },
         { 0x4188e8,   5, 0x04, 0x00000000 },
         { 0x4188fc,   1, 0x04, 0x00100018 },
-        { 0x41891c,   1, 0x04, 0x00ff00ff },
-        { 0x418924,   1, 0x04, 0x00000000 },
-        { 0x418928,   1, 0x04, 0x00ffff00 },
-        { 0x41892c,   1, 0x04, 0x0000ff00 },
-        { 0x418a00,   3, 0x04, 0x00000000 },
-        { 0x418a0c,   1, 0x04, 0x00010000 },
-        { 0x418a10,   3, 0x04, 0x00000000 },
-        { 0x418a20,   3, 0x04, 0x00000000 },
-        { 0x418a2c,   1, 0x04, 0x00010000 },
-        { 0x418a30,   3, 0x04, 0x00000000 },
-        { 0x418a40,   3, 0x04, 0x00000000 },
-        { 0x418a4c,   1, 0x04, 0x00010000 },
-        { 0x418a50,   3, 0x04, 0x00000000 },
-        { 0x418a60,   3, 0x04, 0x00000000 },
-        { 0x418a6c,   1, 0x04, 0x00010000 },
-        { 0x418a70,   3, 0x04, 0x00000000 },
-        { 0x418a80,   3, 0x04, 0x00000000 },
-        { 0x418a8c,   1, 0x04, 0x00010000 },
-        { 0x418a90,   3, 0x04, 0x00000000 },
-        { 0x418aa0,   3, 0x04, 0x00000000 },
-        { 0x418aac,   1, 0x04, 0x00010000 },
-        { 0x418ab0,   3, 0x04, 0x00000000 },
-        { 0x418ac0,   3, 0x04, 0x00000000 },
-        { 0x418acc,   1, 0x04, 0x00010000 },
-        { 0x418ad0,   3, 0x04, 0x00000000 },
-        { 0x418ae0,   3, 0x04, 0x00000000 },
-        { 0x418aec,   1, 0x04, 0x00010000 },
-        { 0x418af0,   3, 0x04, 0x00000000 },
-        { 0x418b00,   1, 0x04, 0x00000000 },
-        { 0x418b08,   1, 0x04, 0x0a418820 },
-        { 0x418b0c,   1, 0x04, 0x062080e6 },
-        { 0x418b10,   1, 0x04, 0x020398a4 },
-        { 0x418b14,   1, 0x04, 0x0e629062 },
-        { 0x418b18,   1, 0x04, 0x0a418820 },
-        { 0x418b1c,   1, 0x04, 0x000000e6 },
-        { 0x418bb8,   1, 0x04, 0x00000103 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc1_grctx_init_gpm_0[] = {
         { 0x418c08,   1, 0x04, 0x00000001 },
         { 0x418c10,   8, 0x04, 0x00000000 },
         { 0x418c6c,   1, 0x04, 0x00000001 },
         { 0x418c80,   1, 0x04, 0x20200004 },
         { 0x418c8c,   1, 0x04, 0x00000001 },
-        { 0x419000,   1, 0x04, 0x00000780 },
-        { 0x419004,   2, 0x04, 0x00000000 },
-        { 0x419014,   1, 0x04, 0x00000004 },
         {}
 };
 
-static struct nvc0_graph_init
-nvc1_grctx_init_tpc[] = {
+static const struct nvc0_graph_pack
+nvc1_grctx_pack_gpc[] = {
+        { nvc0_grctx_init_gpc_unk_0 },
+        { nvc0_grctx_init_prop_0 },
+        { nvc0_grctx_init_gpc_unk_1 },
+        { nvc1_grctx_init_setup_0 },
+        { nvc0_grctx_init_zcull_0 },
+        { nvc0_grctx_init_crstr_0 },
+        { nvc1_grctx_init_gpm_0 },
+        { nvc0_grctx_init_gcc_0 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc1_grctx_init_pe_0[] = {
         { 0x419818,   1, 0x04, 0x00000000 },
         { 0x41983c,   1, 0x04, 0x00038bc7 },
         { 0x419848,   1, 0x04, 0x00000000 },
         { 0x419864,   1, 0x04, 0x00000129 },
         { 0x419888,   1, 0x04, 0x00000000 },
-        { 0x419a00,   1, 0x04, 0x000001f0 },
-        { 0x419a04,   1, 0x04, 0x00000001 },
-        { 0x419a08,   1, 0x04, 0x00000023 },
-        { 0x419a0c,   1, 0x04, 0x00020000 },
-        { 0x419a10,   1, 0x04, 0x00000000 },
-        { 0x419a14,   1, 0x04, 0x00000200 },
-        { 0x419a1c,   1, 0x04, 0x00000000 },
-        { 0x419a20,   1, 0x04, 0x00000800 },
-        { 0x419ac4,   1, 0x04, 0x0007f440 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc1_grctx_init_wwdx_0[] = {
         { 0x419b00,   1, 0x04, 0x0a418820 },
         { 0x419b04,   1, 0x04, 0x062080e6 },
         { 0x419b08,   1, 0x04, 0x020398a4 },
@@ -701,28 +699,33 @@ nvc1_grctx_init_tpc[] = {
         { 0x419bd0,   1, 0x04, 0x00900103 },
         { 0x419be0,   1, 0x04, 0x00400001 },
         { 0x419be4,   1, 0x04, 0x00000000 },
-        { 0x419c00,   1, 0x04, 0x00000002 },
-        { 0x419c04,   1, 0x04, 0x00000006 },
-        { 0x419c08,   1, 0x04, 0x00000002 },
-        { 0x419c20,   1, 0x04, 0x00000000 },
-        { 0x419cb0,   1, 0x04, 0x00020048 },
-        { 0x419ce8,   1, 0x04, 0x00000000 },
-        { 0x419cf4,   1, 0x04, 0x00000183 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc1_grctx_init_tpccs_0[] = {
         { 0x419d20,   1, 0x04, 0x12180000 },
         { 0x419d24,   1, 0x04, 0x00001fff },
         { 0x419d44,   1, 0x04, 0x02180218 },
-        { 0x419e04,   3, 0x04, 0x00000000 },
-        { 0x419e10,   1, 0x04, 0x00000002 },
-        { 0x419e44,   1, 0x04, 0x001beff2 },
-        { 0x419e48,   1, 0x04, 0x00000000 },
-        { 0x419e4c,   1, 0x04, 0x0000000f },
-        { 0x419e50,  17, 0x04, 0x00000000 },
-        { 0x419e98,   1, 0x04, 0x00000000 },
-        { 0x419ee0,   1, 0x04, 0x00011110 },
-        { 0x419f30,  11, 0x04, 0x00000000 },
         {}
 };
 
+static const struct nvc0_graph_pack
+nvc1_grctx_pack_tpc[] = {
+        { nvc1_grctx_init_pe_0 },
+        { nvc4_grctx_init_tex_0 },
+        { nvc1_grctx_init_wwdx_0 },
+        { nvc0_grctx_init_mpc_0 },
+        { nvc4_grctx_init_l1c_0 },
+        { nvc1_grctx_init_tpccs_0 },
+        { nvc4_grctx_init_sm_0 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH context implementation
+ ******************************************************************************/
+
 void
 nvc1_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
 {
@@ -771,41 +774,6 @@ nvc1_grctx_generate_unkn(struct nvc0_graph_priv *priv)
         nv_mask(priv, 0x419c00, 0x00000008, 0x00000008);
 }
 
-static struct nvc0_graph_init *
-nvc1_grctx_init_hub[] = {
-        nvc0_grctx_init_base,
-        nvc0_grctx_init_unk40xx,
-        nvc0_grctx_init_unk44xx,
-        nvc0_grctx_init_unk46xx,
-        nvc0_grctx_init_unk47xx,
-        nvc1_grctx_init_unk58xx,
-        nvc0_grctx_init_unk60xx,
-        nvc0_grctx_init_unk64xx,
-        nvc0_grctx_init_unk78xx,
-        nvc0_grctx_init_unk80xx,
-        nvc1_grctx_init_rop,
-        NULL
-};
-
-struct nvc0_graph_init *
-nvc1_grctx_init_gpc[] = {
-        nvc1_grctx_init_gpc_0,
-        nvc0_grctx_init_gpc_1,
-        nvc1_grctx_init_tpc,
-        NULL
-};
-
-static struct nvc0_graph_mthd
-nvc1_grctx_init_mthd[] = {
-        { 0x9097, nvc1_grctx_init_9097, },
-        { 0x9197, nvc1_grctx_init_9197, },
-        { 0x902d, nvc0_grctx_init_902d, },
-        { 0x9039, nvc0_grctx_init_9039, },
-        { 0x90c0, nvc0_grctx_init_90c0, },
-        { 0x902d, nvc0_grctx_init_mthd_magic, },
-        {}
-};
-
 struct nouveau_oclass *
 nvc1_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .base.handle = NV_ENGCTX(GR, 0xc1),
@@ -817,11 +785,13 @@ nvc1_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .rd32 = _nouveau_graph_context_rd32,
                 .wr32 = _nouveau_graph_context_wr32,
         },
-        .main = nvc0_grctx_generate_main,
-        .mods = nvc1_grctx_generate_mods,
-        .unkn = nvc1_grctx_generate_unkn,
-        .hub  = nvc1_grctx_init_hub,
-        .gpc  = nvc1_grctx_init_gpc,
-        .icmd = nvc1_grctx_init_icmd,
-        .mthd = nvc1_grctx_init_mthd,
+        .main  = nvc0_grctx_generate_main,
+        .mods  = nvc1_grctx_generate_mods,
+        .unkn  = nvc1_grctx_generate_unkn,
+        .hub   = nvc1_grctx_pack_hub,
+        .gpc   = nvc1_grctx_pack_gpc,
+        .zcull = nvc0_grctx_pack_zcull,
+        .tpc   = nvc1_grctx_pack_tpc,
+        .icmd  = nvc1_grctx_pack_icmd,
+        .mthd  = nvc1_grctx_pack_mthd,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc3.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc4.c
index 8f237b3bd8c6..e11ed5538193 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc3.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc4.c
@@ -22,15 +22,14 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
-#include "nvc0.h"
+#include "ctxnvc0.h"
 
-static struct nvc0_graph_init
-nvc3_grctx_init_tpc[] = {
-        { 0x419818,   1, 0x04, 0x00000000 },
-        { 0x41983c,   1, 0x04, 0x00038bc7 },
-        { 0x419848,   1, 0x04, 0x00000000 },
-        { 0x419864,   1, 0x04, 0x0000012a },
-        { 0x419888,   1, 0x04, 0x00000000 },
+/*******************************************************************************
+ * PGRAPH context register lists
+ ******************************************************************************/
+
+const struct nvc0_graph_init
+nvc4_grctx_init_tex_0[] = {
         { 0x419a00,   1, 0x04, 0x000001f0 },
         { 0x419a04,   1, 0x04, 0x00000001 },
         { 0x419a08,   1, 0x04, 0x00000023 },
@@ -40,24 +39,19 @@ nvc3_grctx_init_tpc[] = {
         { 0x419a1c,   1, 0x04, 0x00000000 },
         { 0x419a20,   1, 0x04, 0x00000800 },
         { 0x419ac4,   1, 0x04, 0x0007f440 },
-        { 0x419b00,   1, 0x04, 0x0a418820 },
-        { 0x419b04,   1, 0x04, 0x062080e6 },
-        { 0x419b08,   1, 0x04, 0x020398a4 },
-        { 0x419b0c,   1, 0x04, 0x0e629062 },
-        { 0x419b10,   1, 0x04, 0x0a418820 },
-        { 0x419b14,   1, 0x04, 0x000000e6 },
-        { 0x419bd0,   1, 0x04, 0x00900103 },
-        { 0x419be0,   1, 0x04, 0x00000001 },
-        { 0x419be4,   1, 0x04, 0x00000000 },
-        { 0x419c00,   1, 0x04, 0x00000002 },
-        { 0x419c04,   1, 0x04, 0x00000006 },
-        { 0x419c08,   1, 0x04, 0x00000002 },
-        { 0x419c20,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc4_grctx_init_l1c_0[] = {
         { 0x419cb0,   1, 0x04, 0x00020048 },
         { 0x419ce8,   1, 0x04, 0x00000000 },
         { 0x419cf4,   1, 0x04, 0x00000183 },
-        { 0x419d20,   1, 0x04, 0x02180000 },
-        { 0x419d24,   1, 0x04, 0x00001fff },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc4_grctx_init_sm_0[] = {
         { 0x419e04,   3, 0x04, 0x00000000 },
         { 0x419e10,   1, 0x04, 0x00000002 },
         { 0x419e44,   1, 0x04, 0x001beff2 },
@@ -70,16 +64,24 @@ nvc3_grctx_init_tpc[] = {
         {}
 };
 
-struct nvc0_graph_init *
-nvc3_grctx_init_gpc[] = {
-        nvc0_grctx_init_gpc_0,
-        nvc0_grctx_init_gpc_1,
-        nvc3_grctx_init_tpc,
-        NULL
+static const struct nvc0_graph_pack
+nvc4_grctx_pack_tpc[] = {
+        { nvc0_grctx_init_pe_0 },
+        { nvc4_grctx_init_tex_0 },
+        { nvc0_grctx_init_wwdx_0 },
+        { nvc0_grctx_init_mpc_0 },
+        { nvc4_grctx_init_l1c_0 },
+        { nvc0_grctx_init_tpccs_0 },
+        { nvc4_grctx_init_sm_0 },
+        {}
 };
 
+/*******************************************************************************
+ * PGRAPH context implementation
+ ******************************************************************************/
+
 struct nouveau_oclass *
-nvc3_grctx_oclass = &(struct nvc0_grctx_oclass) {
+nvc4_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .base.handle = NV_ENGCTX(GR, 0xc3),
         .base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nvc0_graph_context_ctor,
@@ -89,11 +91,13 @@ nvc3_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .rd32 = _nouveau_graph_context_rd32,
                 .wr32 = _nouveau_graph_context_wr32,
         },
-        .main = nvc0_grctx_generate_main,
-        .mods = nvc0_grctx_generate_mods,
-        .unkn = nvc0_grctx_generate_unkn,
-        .hub  = nvc0_grctx_init_hub,
-        .gpc  = nvc3_grctx_init_gpc,
-        .icmd = nvc0_grctx_init_icmd,
-        .mthd = nvc0_grctx_init_mthd,
+        .main  = nvc0_grctx_generate_main,
+        .mods  = nvc0_grctx_generate_mods,
+        .unkn  = nvc0_grctx_generate_unkn,
+        .hub   = nvc0_grctx_pack_hub,
+        .gpc   = nvc0_grctx_pack_gpc,
+        .zcull = nvc0_grctx_pack_zcull,
+        .tpc   = nvc4_grctx_pack_tpc,
+        .icmd  = nvc0_grctx_pack_icmd,
+        .mthd  = nvc0_grctx_pack_mthd,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc8.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc8.c
index d0d4ce3c4892..feebd58dfe8d 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc8.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvc8.c
@@ -22,10 +22,14 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
-#include "nvc0.h"
+#include "ctxnvc0.h"
 
-static struct nvc0_graph_init
-nvc8_grctx_init_icmd[] = {
+/*******************************************************************************
+ * PGRAPH context register lists
+ ******************************************************************************/
+
+static const struct nvc0_graph_init
+nvc8_grctx_init_icmd_0[] = {
         { 0x001000,   1, 0x01, 0x00000004 },
         { 0x0000a9,   1, 0x01, 0x0000ffff },
         { 0x000038,   1, 0x01, 0x0fac6881 },
@@ -141,8 +145,7 @@ nvc8_grctx_init_icmd[] = {
         { 0x000586,   1, 0x01, 0x00000040 },
         { 0x000582,   2, 0x01, 0x00000080 },
         { 0x0005c2,   1, 0x01, 0x00000001 },
-        { 0x000638,   1, 0x01, 0x00000001 },
-        { 0x000639,   1, 0x01, 0x00000001 },
+        { 0x000638,   2, 0x01, 0x00000001 },
         { 0x00063a,   1, 0x01, 0x00000002 },
         { 0x00063b,   2, 0x01, 0x00000001 },
         { 0x00063d,   1, 0x01, 0x00000002 },
@@ -203,15 +206,13 @@ nvc8_grctx_init_icmd[] = {
         { 0x000787,   1, 0x01, 0x000000cf },
         { 0x00078c,   1, 0x01, 0x00000008 },
         { 0x000792,   1, 0x01, 0x00000001 },
-        { 0x000794,   1, 0x01, 0x00000001 },
-        { 0x000795,   2, 0x01, 0x00000001 },
+        { 0x000794,   3, 0x01, 0x00000001 },
         { 0x000797,   1, 0x01, 0x000000cf },
         { 0x000836,   1, 0x01, 0x00000001 },
         { 0x00079a,   1, 0x01, 0x00000002 },
         { 0x000833,   1, 0x01, 0x04444480 },
         { 0x0007a1,   1, 0x01, 0x00000001 },
-        { 0x0007a3,   1, 0x01, 0x00000001 },
-        { 0x0007a4,   2, 0x01, 0x00000001 },
+        { 0x0007a3,   3, 0x01, 0x00000001 },
         { 0x000831,   1, 0x01, 0x00000004 },
         { 0x00080c,   1, 0x01, 0x00000002 },
         { 0x00080d,   2, 0x01, 0x00000100 },
@@ -237,14 +238,12 @@ nvc8_grctx_init_icmd[] = {
         { 0x0006b1,   1, 0x01, 0x00000011 },
         { 0x00078c,   1, 0x01, 0x00000008 },
         { 0x000792,   1, 0x01, 0x00000001 },
-        { 0x000794,   1, 0x01, 0x00000001 },
-        { 0x000795,   2, 0x01, 0x00000001 },
+        { 0x000794,   3, 0x01, 0x00000001 },
         { 0x000797,   1, 0x01, 0x000000cf },
         { 0x00079a,   1, 0x01, 0x00000002 },
         { 0x000833,   1, 0x01, 0x04444480 },
         { 0x0007a1,   1, 0x01, 0x00000001 },
-        { 0x0007a3,   1, 0x01, 0x00000001 },
-        { 0x0007a4,   2, 0x01, 0x00000001 },
+        { 0x0007a3,   3, 0x01, 0x00000001 },
         { 0x000831,   1, 0x01, 0x00000004 },
         { 0x01e100,   1, 0x01, 0x00000001 },
         { 0x001000,   1, 0x01, 0x00000014 },
@@ -269,58 +268,20 @@ nvc8_grctx_init_icmd[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nvc8_grctx_init_tpc[] = {
-        { 0x419818,   1, 0x04, 0x00000000 },
-        { 0x41983c,   1, 0x04, 0x00038bc7 },
-        { 0x419848,   1, 0x04, 0x00000000 },
-        { 0x419864,   1, 0x04, 0x0000012a },
-        { 0x419888,   1, 0x04, 0x00000000 },
-        { 0x419a00,   1, 0x04, 0x000001f0 },
-        { 0x419a04,   1, 0x04, 0x00000001 },
-        { 0x419a08,   1, 0x04, 0x00000023 },
-        { 0x419a0c,   1, 0x04, 0x00020000 },
-        { 0x419a10,   1, 0x04, 0x00000000 },
-        { 0x419a14,   1, 0x04, 0x00000200 },
-        { 0x419a1c,   1, 0x04, 0x00000000 },
-        { 0x419a20,   1, 0x04, 0x00000800 },
-        { 0x419b00,   1, 0x04, 0x0a418820 },
-        { 0x419b04,   1, 0x04, 0x062080e6 },
-        { 0x419b08,   1, 0x04, 0x020398a4 },
-        { 0x419b0c,   1, 0x04, 0x0e629062 },
-        { 0x419b10,   1, 0x04, 0x0a418820 },
-        { 0x419b14,   1, 0x04, 0x000000e6 },
-        { 0x419bd0,   1, 0x04, 0x00900103 },
-        { 0x419be0,   1, 0x04, 0x00000001 },
-        { 0x419be4,   1, 0x04, 0x00000000 },
-        { 0x419c00,   1, 0x04, 0x00000002 },
-        { 0x419c04,   1, 0x04, 0x00000006 },
-        { 0x419c08,   1, 0x04, 0x00000002 },
-        { 0x419c20,   1, 0x04, 0x00000000 },
-        { 0x419cb0,   1, 0x04, 0x00060048 },
-        { 0x419ce8,   1, 0x04, 0x00000000 },
-        { 0x419cf4,   1, 0x04, 0x00000183 },
-        { 0x419d20,   1, 0x04, 0x02180000 },
-        { 0x419d24,   1, 0x04, 0x00001fff },
-        { 0x419e04,   3, 0x04, 0x00000000 },
-        { 0x419e10,   1, 0x04, 0x00000002 },
-        { 0x419e44,   1, 0x04, 0x001beff2 },
-        { 0x419e48,   1, 0x04, 0x00000000 },
-        { 0x419e4c,   1, 0x04, 0x0000000f },
-        { 0x419e50,  17, 0x04, 0x00000000 },
-        { 0x419e98,   1, 0x04, 0x00000000 },
-        { 0x419f50,   2, 0x04, 0x00000000 },
+static const struct nvc0_graph_pack
+nvc8_grctx_pack_icmd[] = {
+        { nvc8_grctx_init_icmd_0 },
         {}
 };
 
-struct nvc0_graph_init
-nvc8_grctx_init_9197[] = {
+const struct nvc0_graph_init
+nvc8_grctx_init_9197_0[] = {
         { 0x0002e4,   1, 0x04, 0x0000b001 },
         {}
 };
 
-struct nvc0_graph_init
-nvc8_grctx_init_9297[] = {
+const struct nvc0_graph_init
+nvc8_grctx_init_9297_0[] = {
         { 0x003400, 128, 0x04, 0x00000000 },
         { 0x00036c,   2, 0x04, 0x00000000 },
         { 0x0007a4,   2, 0x04, 0x00000000 },
@@ -329,26 +290,47 @@ nvc8_grctx_init_9297[] = {
         {}
 };
 
-static struct nvc0_graph_mthd
-nvc8_grctx_init_mthd[] = {
-        { 0x9097, nvc1_grctx_init_9097, },
-        { 0x9197, nvc8_grctx_init_9197, },
-        { 0x9297, nvc8_grctx_init_9297, },
-        { 0x902d, nvc0_grctx_init_902d, },
-        { 0x9039, nvc0_grctx_init_9039, },
-        { 0x90c0, nvc0_grctx_init_90c0, },
-        { 0x902d, nvc0_grctx_init_mthd_magic, },
+static const struct nvc0_graph_pack
+nvc8_grctx_pack_mthd[] = {
+        { nvc1_grctx_init_9097_0, 0x9097 },
+        { nvc8_grctx_init_9197_0, 0x9197 },
+        { nvc8_grctx_init_9297_0, 0x9297 },
+        { nvc0_grctx_init_902d_0, 0x902d },
+        { nvc0_grctx_init_9039_0, 0x9039 },
+        { nvc0_grctx_init_90c0_0, 0x90c0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvc8_grctx_init_setup_0[] = {
+        { 0x418800,   1, 0x04, 0x0006860a },
+        { 0x418808,   3, 0x04, 0x00000000 },
+        { 0x418828,   1, 0x04, 0x00008442 },
+        { 0x418830,   1, 0x04, 0x00000001 },
+        { 0x4188d8,   1, 0x04, 0x00000008 },
+        { 0x4188e0,   1, 0x04, 0x01000000 },
+        { 0x4188e8,   5, 0x04, 0x00000000 },
+        { 0x4188fc,   1, 0x04, 0x20100000 },
         {}
 };
 
-static struct nvc0_graph_init *
-nvc8_grctx_init_gpc[] = {
-        nvc0_grctx_init_gpc_0,
-        nvc0_grctx_init_gpc_1,
-        nvc8_grctx_init_tpc,
-        NULL
+static const struct nvc0_graph_pack
+nvc8_grctx_pack_gpc[] = {
+        { nvc0_grctx_init_gpc_unk_0 },
+        { nvc0_grctx_init_prop_0 },
+        { nvc0_grctx_init_gpc_unk_1 },
+        { nvc8_grctx_init_setup_0 },
+        { nvc0_grctx_init_zcull_0 },
+        { nvc0_grctx_init_crstr_0 },
+        { nvc0_grctx_init_gpm_0 },
+        { nvc0_grctx_init_gcc_0 },
+        {}
 };
 
+/*******************************************************************************
+ * PGRAPH context implementation
+ ******************************************************************************/
+
 struct nouveau_oclass *
 nvc8_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .base.handle = NV_ENGCTX(GR, 0xc8),
@@ -360,11 +342,13 @@ nvc8_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .rd32 = _nouveau_graph_context_rd32,
                 .wr32 = _nouveau_graph_context_wr32,
         },
-        .main = nvc0_grctx_generate_main,
-        .mods = nvc0_grctx_generate_mods,
-        .unkn = nvc0_grctx_generate_unkn,
-        .hub  = nvc0_grctx_init_hub,
-        .gpc  = nvc8_grctx_init_gpc,
-        .icmd = nvc8_grctx_init_icmd,
-        .mthd = nvc8_grctx_init_mthd,
+        .main  = nvc0_grctx_generate_main,
+        .mods  = nvc0_grctx_generate_mods,
+        .unkn  = nvc0_grctx_generate_unkn,
+        .hub   = nvc0_grctx_pack_hub,
+        .gpc   = nvc8_grctx_pack_gpc,
+        .zcull = nvc0_grctx_pack_zcull,
+        .tpc   = nvc0_grctx_pack_tpc,
+        .icmd  = nvc8_grctx_pack_icmd,
+        .mthd  = nvc8_grctx_pack_mthd,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd7.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd7.c
index c4740d528532..1dbc8d7f2e86 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd7.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd7.c
@@ -22,33 +22,14 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
-#include "nvc0.h"
+#include "ctxnvc0.h"
 
-struct nvc0_graph_init
-nvd7_grctx_init_unk40xx[] = {
-        { 0x404004,  10, 0x04, 0x00000000 },
-        { 0x404044,   1, 0x04, 0x00000000 },
-        { 0x404094,   1, 0x04, 0x00000000 },
-        { 0x404098,  12, 0x04, 0x00000000 },
-        { 0x4040c8,   1, 0x04, 0xf0000087 },
-        { 0x4040d0,   6, 0x04, 0x00000000 },
-        { 0x4040e8,   1, 0x04, 0x00001000 },
-        { 0x4040f8,   1, 0x04, 0x00000000 },
-        { 0x404130,   1, 0x04, 0x00000000 },
-        { 0x404134,   1, 0x04, 0x00000000 },
-        { 0x404138,   1, 0x04, 0x20000040 },
-        { 0x404150,   1, 0x04, 0x0000002e },
-        { 0x404154,   1, 0x04, 0x00000400 },
-        { 0x404158,   1, 0x04, 0x00000200 },
-        { 0x404164,   1, 0x04, 0x00000055 },
-        { 0x404168,   1, 0x04, 0x00000000 },
-        { 0x404178,   2, 0x04, 0x00000000 },
-        { 0x404200,   8, 0x04, 0x00000000 },
-        {}
-};
+/*******************************************************************************
+ * PGRAPH context register lists
+ ******************************************************************************/
 
-static struct nvc0_graph_init
-nvd7_grctx_init_unk58xx[] = {
+static const struct nvc0_graph_init
+nvd7_grctx_init_ds_0[] = {
         { 0x405800,   1, 0x04, 0x0f8000bf },
         { 0x405830,   1, 0x04, 0x02180324 },
         { 0x405834,   1, 0x04, 0x08000000 },
@@ -60,8 +41,10 @@ nvd7_grctx_init_unk58xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nvd7_grctx_init_unk64xx[] = {
+static const struct nvc0_graph_init
+nvd7_grctx_init_pd_0[] = {
+        { 0x406020,   1, 0x04, 0x000103c1 },
+        { 0x406028,   4, 0x04, 0x00000001 },
         { 0x4064a8,   1, 0x04, 0x00000000 },
         { 0x4064ac,   1, 0x04, 0x00003fff },
         { 0x4064b4,   3, 0x04, 0x00000000 },
@@ -71,22 +54,22 @@ nvd7_grctx_init_unk64xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nvd7_grctx_init_gpc_0[] = {
-        { 0x418380,   1, 0x04, 0x00000016 },
-        { 0x418400,   1, 0x04, 0x38004e00 },
-        { 0x418404,   1, 0x04, 0x71e0ffff },
-        { 0x41840c,   1, 0x04, 0x00001008 },
-        { 0x418410,   1, 0x04, 0x0fff0fff },
-        { 0x418414,   1, 0x04, 0x02200fff },
-        { 0x418450,   6, 0x04, 0x00000000 },
-        { 0x418468,   1, 0x04, 0x00000001 },
-        { 0x41846c,   2, 0x04, 0x00000000 },
-        { 0x418600,   1, 0x04, 0x0000001f },
-        { 0x418684,   1, 0x04, 0x0000000f },
-        { 0x418700,   1, 0x04, 0x00000002 },
-        { 0x418704,   1, 0x04, 0x00000080 },
-        { 0x418708,   3, 0x04, 0x00000000 },
+static const struct nvc0_graph_pack
+nvd7_grctx_pack_hub[] = {
+        { nvc0_grctx_init_main_0 },
+        { nvd9_grctx_init_fe_0 },
+        { nvc0_grctx_init_pri_0 },
+        { nvc0_grctx_init_memfmt_0 },
+        { nvd7_grctx_init_ds_0 },
+        { nvd7_grctx_init_pd_0 },
+        { nvc0_grctx_init_rstr2d_0 },
+        { nvc0_grctx_init_scc_0 },
+        { nvd9_grctx_init_be_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvd7_grctx_init_setup_0[] = {
         { 0x418800,   1, 0x04, 0x7006860a },
         { 0x418808,   3, 0x04, 0x00000000 },
         { 0x418828,   1, 0x04, 0x00008442 },
@@ -95,34 +78,32 @@ nvd7_grctx_init_gpc_0[] = {
         { 0x4188e0,   1, 0x04, 0x01000000 },
         { 0x4188e8,   5, 0x04, 0x00000000 },
         { 0x4188fc,   1, 0x04, 0x20100018 },
-        { 0x41891c,   1, 0x04, 0x00ff00ff },
-        { 0x418924,   1, 0x04, 0x00000000 },
-        { 0x418928,   1, 0x04, 0x00ffff00 },
-        { 0x41892c,   1, 0x04, 0x0000ff00 },
-        { 0x418b00,   1, 0x04, 0x00000006 },
-        { 0x418b08,   1, 0x04, 0x0a418820 },
-        { 0x418b0c,   1, 0x04, 0x062080e6 },
-        { 0x418b10,   1, 0x04, 0x020398a4 },
-        { 0x418b14,   1, 0x04, 0x0e629062 },
-        { 0x418b18,   1, 0x04, 0x0a418820 },
-        { 0x418b1c,   1, 0x04, 0x000000e6 },
-        { 0x418bb8,   1, 0x04, 0x00000103 },
-        { 0x418c08,   1, 0x04, 0x00000001 },
-        { 0x418c10,   8, 0x04, 0x00000000 },
-        { 0x418c6c,   1, 0x04, 0x00000001 },
-        { 0x418c80,   1, 0x04, 0x20200004 },
-        { 0x418c8c,   1, 0x04, 0x00000001 },
-        { 0x419000,   1, 0x04, 0x00000780 },
-        { 0x419004,   2, 0x04, 0x00000000 },
-        { 0x419014,   1, 0x04, 0x00000004 },
         {}
 };
 
-static struct nvc0_graph_init
-nvd7_grctx_init_tpc[] = {
+static const struct nvc0_graph_pack
+nvd7_grctx_pack_gpc[] = {
+        { nvc0_grctx_init_gpc_unk_0 },
+        { nvd9_grctx_init_prop_0 },
+        { nvd9_grctx_init_gpc_unk_1 },
+        { nvd7_grctx_init_setup_0 },
+        { nvc0_grctx_init_zcull_0 },
+        { nvd9_grctx_init_crstr_0 },
+        { nvc1_grctx_init_gpm_0 },
+        { nvc0_grctx_init_gcc_0 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvd7_grctx_init_pe_0[] = {
         { 0x419848,   1, 0x04, 0x00000000 },
         { 0x419864,   1, 0x04, 0x00000129 },
         { 0x419888,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvd7_grctx_init_tex_0[] = {
         { 0x419a00,   1, 0x04, 0x000001f0 },
         { 0x419a04,   1, 0x04, 0x00000001 },
         { 0x419a08,   1, 0x04, 0x00000023 },
@@ -132,33 +113,46 @@ nvd7_grctx_init_tpc[] = {
         { 0x419a1c,   1, 0x04, 0x00008000 },
         { 0x419a20,   1, 0x04, 0x00000800 },
         { 0x419ac4,   1, 0x04, 0x0017f440 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvd7_grctx_init_mpc_0[] = {
         { 0x419c00,   1, 0x04, 0x0000000a },
         { 0x419c04,   1, 0x04, 0x00000006 },
         { 0x419c08,   1, 0x04, 0x00000002 },
         { 0x419c20,   1, 0x04, 0x00000000 },
         { 0x419c24,   1, 0x04, 0x00084210 },
         { 0x419c28,   1, 0x04, 0x3efbefbe },
-        { 0x419cb0,   1, 0x04, 0x00020048 },
-        { 0x419ce8,   1, 0x04, 0x00000000 },
-        { 0x419cf4,   1, 0x04, 0x00000183 },
-        { 0x419e04,   3, 0x04, 0x00000000 },
-        { 0x419e10,   1, 0x04, 0x00000002 },
-        { 0x419e44,   1, 0x04, 0x001beff2 },
-        { 0x419e48,   1, 0x04, 0x00000000 },
-        { 0x419e4c,   1, 0x04, 0x0000000f },
-        { 0x419e50,  17, 0x04, 0x00000000 },
-        { 0x419e98,   1, 0x04, 0x00000000 },
-        { 0x419ee0,   1, 0x04, 0x00010110 },
-        { 0x419f30,  11, 0x04, 0x00000000 },
         {}
 };
 
-static struct nvc0_graph_init
-nvd7_grctx_init_unk[] = {
+static const struct nvc0_graph_pack
+nvd7_grctx_pack_tpc[] = {
+        { nvd7_grctx_init_pe_0 },
+        { nvd7_grctx_init_tex_0 },
+        { nvd7_grctx_init_mpc_0 },
+        { nvc4_grctx_init_l1c_0 },
+        { nvd9_grctx_init_sm_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvd7_grctx_init_pes_0[] = {
         { 0x41be24,   1, 0x04, 0x00000002 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvd7_grctx_init_cbm_0[] = {
         { 0x41bec0,   1, 0x04, 0x12180000 },
         { 0x41bec4,   1, 0x04, 0x00003fff },
         { 0x41bee4,   1, 0x04, 0x03240218 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvd7_grctx_init_wwdx_0[] = {
         { 0x41bf00,   1, 0x04, 0x0a418820 },
         { 0x41bf04,   1, 0x04, 0x062080e6 },
         { 0x41bf08,   1, 0x04, 0x020398a4 },
@@ -171,6 +165,18 @@ nvd7_grctx_init_unk[] = {
         {}
 };
 
+static const struct nvc0_graph_pack
+nvd7_grctx_pack_ppc[] = {
+        { nvd7_grctx_init_pes_0 },
+        { nvd7_grctx_init_cbm_0 },
+        { nvd7_grctx_init_wwdx_0 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH context implementation
+ ******************************************************************************/
+
 static void
 nvd7_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
 {
@@ -219,10 +225,11 @@ nvd7_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
 
         nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
 
-        for (i = 0; oclass->hub[i]; i++)
-                nvc0_graph_mmio(priv, oclass->hub[i]);
-        for (i = 0; oclass->gpc[i]; i++)
-                nvc0_graph_mmio(priv, oclass->gpc[i]);
+        nvc0_graph_mmio(priv, oclass->hub);
+        nvc0_graph_mmio(priv, oclass->gpc);
+        nvc0_graph_mmio(priv, oclass->zcull);
+        nvc0_graph_mmio(priv, oclass->tpc);
+        nvc0_graph_mmio(priv, oclass->ppc);
 
         nv_wr32(priv, 0x404154, 0x00000000);
 
@@ -244,32 +251,6 @@ nvd7_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
         nv_mask(priv, 0x000260, 0x00000001, 0x00000001);
 }
 
-
-static struct nvc0_graph_init *
-nvd7_grctx_init_hub[] = {
-        nvc0_grctx_init_base,
-        nvd7_grctx_init_unk40xx,
-        nvc0_grctx_init_unk44xx,
-        nvc0_grctx_init_unk46xx,
-        nvc0_grctx_init_unk47xx,
-        nvd7_grctx_init_unk58xx,
-        nvc0_grctx_init_unk60xx,
-        nvd7_grctx_init_unk64xx,
-        nvc0_grctx_init_unk78xx,
-        nvc0_grctx_init_unk80xx,
-        nvd9_grctx_init_rop,
-        NULL
-};
-
-struct nvc0_graph_init *
-nvd7_grctx_init_gpc[] = {
-        nvd7_grctx_init_gpc_0,
-        nvc0_grctx_init_gpc_1,
-        nvd7_grctx_init_tpc,
-        nvd7_grctx_init_unk,
-        NULL
-};
-
 struct nouveau_oclass *
 nvd7_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .base.handle = NV_ENGCTX(GR, 0xd7),
@@ -281,11 +262,14 @@ nvd7_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .rd32 = _nouveau_graph_context_rd32,
                 .wr32 = _nouveau_graph_context_wr32,
         },
-        .main = nvd7_grctx_generate_main,
-        .mods = nvd7_grctx_generate_mods,
-        .unkn = nve4_grctx_generate_unkn,
-        .hub  = nvd7_grctx_init_hub,
-        .gpc  = nvd7_grctx_init_gpc,
-        .icmd = nvd9_grctx_init_icmd,
-        .mthd = nvd9_grctx_init_mthd,
+        .main  = nvd7_grctx_generate_main,
+        .mods  = nvd7_grctx_generate_mods,
+        .unkn  = nve4_grctx_generate_unkn,
+        .hub   = nvd7_grctx_pack_hub,
+        .gpc   = nvd7_grctx_pack_gpc,
+        .zcull = nvc0_grctx_pack_zcull,
+        .tpc   = nvd7_grctx_pack_tpc,
+        .ppc   = nvd7_grctx_pack_ppc,
+        .icmd  = nvd9_grctx_pack_icmd,
+        .mthd  = nvd9_grctx_pack_mthd,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd9.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd9.c
index a1102cbf2fdc..c665fb7e4660 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd9.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvd9.c
@@ -22,38 +22,14 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
-#include "nvc0.h"
+#include "ctxnvc0.h"
 
-struct nvc0_graph_init
-nvd9_grctx_init_90c0[] = {
-        { 0x002700,   4, 0x40, 0x00000000 },
-        { 0x002720,   4, 0x40, 0x00000000 },
-        { 0x002704,   4, 0x40, 0x00000000 },
-        { 0x002724,   4, 0x40, 0x00000000 },
-        { 0x002708,   4, 0x40, 0x00000000 },
-        { 0x002728,   4, 0x40, 0x00000000 },
-        { 0x00270c,   8, 0x20, 0x00000000 },
-        { 0x002710,   4, 0x40, 0x00014000 },
-        { 0x002730,   4, 0x40, 0x00014000 },
-        { 0x002714,   4, 0x40, 0x00000040 },
-        { 0x002734,   4, 0x40, 0x00000040 },
-        { 0x00030c,   1, 0x04, 0x00000001 },
-        { 0x001944,   1, 0x04, 0x00000000 },
-        { 0x000758,   1, 0x04, 0x00000100 },
-        { 0x0002c4,   1, 0x04, 0x00000000 },
-        { 0x000790,   5, 0x04, 0x00000000 },
-        { 0x00077c,   1, 0x04, 0x00000000 },
-        { 0x000204,   3, 0x04, 0x00000000 },
-        { 0x000214,   1, 0x04, 0x00000000 },
-        { 0x00024c,   1, 0x04, 0x00000000 },
-        { 0x000d94,   1, 0x04, 0x00000001 },
-        { 0x001608,   2, 0x04, 0x00000000 },
-        { 0x001664,   1, 0x04, 0x00000000 },
-        {}
-};
+/*******************************************************************************
+ * PGRAPH context register lists
+ ******************************************************************************/
 
-struct nvc0_graph_init
-nvd9_grctx_init_icmd[] = {
+static const struct nvc0_graph_init
+nvd9_grctx_init_icmd_0[] = {
         { 0x001000,   1, 0x01, 0x00000004 },
         { 0x0000a9,   1, 0x01, 0x0000ffff },
         { 0x000038,   1, 0x01, 0x0fac6881 },
@@ -171,8 +147,7 @@ nvd9_grctx_init_icmd[] = {
         { 0x000586,   1, 0x01, 0x00000040 },
         { 0x000582,   2, 0x01, 0x00000080 },
         { 0x0005c2,   1, 0x01, 0x00000001 },
-        { 0x000638,   1, 0x01, 0x00000001 },
-        { 0x000639,   1, 0x01, 0x00000001 },
+        { 0x000638,   2, 0x01, 0x00000001 },
         { 0x00063a,   1, 0x01, 0x00000002 },
         { 0x00063b,   2, 0x01, 0x00000001 },
         { 0x00063d,   1, 0x01, 0x00000002 },
@@ -233,15 +208,13 @@ nvd9_grctx_init_icmd[] = {
         { 0x000787,   1, 0x01, 0x000000cf },
         { 0x00078c,   1, 0x01, 0x00000008 },
         { 0x000792,   1, 0x01, 0x00000001 },
-        { 0x000794,   1, 0x01, 0x00000001 },
-        { 0x000795,   2, 0x01, 0x00000001 },
+        { 0x000794,   3, 0x01, 0x00000001 },
         { 0x000797,   1, 0x01, 0x000000cf },
         { 0x000836,   1, 0x01, 0x00000001 },
         { 0x00079a,   1, 0x01, 0x00000002 },
         { 0x000833,   1, 0x01, 0x04444480 },
         { 0x0007a1,   1, 0x01, 0x00000001 },
-        { 0x0007a3,   1, 0x01, 0x00000001 },
-        { 0x0007a4,   2, 0x01, 0x00000001 },
+        { 0x0007a3,   3, 0x01, 0x00000001 },
         { 0x000831,   1, 0x01, 0x00000004 },
         { 0x00080c,   1, 0x01, 0x00000002 },
         { 0x00080d,   2, 0x01, 0x00000100 },
@@ -267,14 +240,12 @@ nvd9_grctx_init_icmd[] = {
         { 0x0006b1,   1, 0x01, 0x00000011 },
         { 0x00078c,   1, 0x01, 0x00000008 },
         { 0x000792,   1, 0x01, 0x00000001 },
-        { 0x000794,   1, 0x01, 0x00000001 },
-        { 0x000795,   2, 0x01, 0x00000001 },
+        { 0x000794,   3, 0x01, 0x00000001 },
         { 0x000797,   1, 0x01, 0x000000cf },
         { 0x00079a,   1, 0x01, 0x00000002 },
         { 0x000833,   1, 0x01, 0x04444480 },
         { 0x0007a1,   1, 0x01, 0x00000001 },
-        { 0x0007a3,   1, 0x01, 0x00000001 },
-        { 0x0007a4,   2, 0x01, 0x00000001 },
+        { 0x0007a3,   3, 0x01, 0x00000001 },
         { 0x000831,   1, 0x01, 0x00000004 },
         { 0x01e100,   1, 0x01, 0x00000001 },
         { 0x001000,   1, 0x01, 0x00000014 },
@@ -299,18 +270,56 @@ nvd9_grctx_init_icmd[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvd9_grctx_init_unk40xx[] = {
-        { 0x404004,  11, 0x04, 0x00000000 },
+const struct nvc0_graph_pack
+nvd9_grctx_pack_icmd[] = {
+        { nvd9_grctx_init_icmd_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvd9_grctx_init_90c0_0[] = {
+        { 0x002700,   8, 0x20, 0x00000000 },
+        { 0x002704,   8, 0x20, 0x00000000 },
+        { 0x002708,   8, 0x20, 0x00000000 },
+        { 0x00270c,   8, 0x20, 0x00000000 },
+        { 0x002710,   8, 0x20, 0x00014000 },
+        { 0x002714,   8, 0x20, 0x00000040 },
+        { 0x00030c,   1, 0x04, 0x00000001 },
+        { 0x001944,   1, 0x04, 0x00000000 },
+        { 0x000758,   1, 0x04, 0x00000100 },
+        { 0x0002c4,   1, 0x04, 0x00000000 },
+        { 0x000790,   5, 0x04, 0x00000000 },
+        { 0x00077c,   1, 0x04, 0x00000000 },
+        { 0x000204,   3, 0x04, 0x00000000 },
+        { 0x000214,   1, 0x04, 0x00000000 },
+        { 0x00024c,   1, 0x04, 0x00000000 },
+        { 0x000d94,   1, 0x04, 0x00000001 },
+        { 0x001608,   2, 0x04, 0x00000000 },
+        { 0x001664,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_pack
+nvd9_grctx_pack_mthd[] = {
+        { nvc1_grctx_init_9097_0, 0x9097 },
+        { nvc8_grctx_init_9197_0, 0x9197 },
+        { nvc8_grctx_init_9297_0, 0x9297 },
+        { nvc0_grctx_init_902d_0, 0x902d },
+        { nvc0_grctx_init_9039_0, 0x9039 },
+        { nvd9_grctx_init_90c0_0, 0x90c0 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvd9_grctx_init_fe_0[] = {
+        { 0x404004,  10, 0x04, 0x00000000 },
         { 0x404044,   1, 0x04, 0x00000000 },
-        { 0x404094,   1, 0x04, 0x00000000 },
-        { 0x404098,  12, 0x04, 0x00000000 },
+        { 0x404094,  13, 0x04, 0x00000000 },
         { 0x4040c8,   1, 0x04, 0xf0000087 },
         { 0x4040d0,   6, 0x04, 0x00000000 },
         { 0x4040e8,   1, 0x04, 0x00001000 },
         { 0x4040f8,   1, 0x04, 0x00000000 },
-        { 0x404130,   1, 0x04, 0x00000000 },
-        { 0x404134,   1, 0x04, 0x00000000 },
+        { 0x404130,   2, 0x04, 0x00000000 },
         { 0x404138,   1, 0x04, 0x20000040 },
         { 0x404150,   1, 0x04, 0x0000002e },
         { 0x404154,   1, 0x04, 0x00000400 },
@@ -322,8 +331,8 @@ nvd9_grctx_init_unk40xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nvd9_grctx_init_unk58xx[] = {
+static const struct nvc0_graph_init
+nvd9_grctx_init_ds_0[] = {
         { 0x405800,   1, 0x04, 0x0f8000bf },
         { 0x405830,   1, 0x04, 0x02180218 },
         { 0x405834,   1, 0x04, 0x08000000 },
@@ -335,8 +344,10 @@ nvd9_grctx_init_unk58xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nvd9_grctx_init_unk64xx[] = {
+static const struct nvc0_graph_init
+nvd9_grctx_init_pd_0[] = {
+        { 0x406020,   1, 0x04, 0x000103c1 },
+        { 0x406028,   4, 0x04, 0x00000001 },
         { 0x4064a8,   1, 0x04, 0x00000000 },
         { 0x4064ac,   1, 0x04, 0x00003fff },
         { 0x4064b4,   3, 0x04, 0x00000000 },
@@ -345,21 +356,34 @@ nvd9_grctx_init_unk64xx[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvd9_grctx_init_rop[] = {
+const struct nvc0_graph_init
+nvd9_grctx_init_be_0[] = {
         { 0x408800,   1, 0x04, 0x02802a3c },
         { 0x408804,   1, 0x04, 0x00000040 },
         { 0x408808,   1, 0x04, 0x1043e005 },
         { 0x408900,   1, 0x04, 0x3080b801 },
-        { 0x408904,   1, 0x04, 0x1043e005 },
+        { 0x408904,   1, 0x04, 0x62000001 },
         { 0x408908,   1, 0x04, 0x00c8102f },
         { 0x408980,   1, 0x04, 0x0000011d },
         {}
 };
 
-static struct nvc0_graph_init
-nvd9_grctx_init_gpc_0[] = {
-        { 0x418380,   1, 0x04, 0x00000016 },
+static const struct nvc0_graph_pack
+nvd9_grctx_pack_hub[] = {
+        { nvc0_grctx_init_main_0 },
+        { nvd9_grctx_init_fe_0 },
+        { nvc0_grctx_init_pri_0 },
+        { nvc0_grctx_init_memfmt_0 },
+        { nvd9_grctx_init_ds_0 },
+        { nvd9_grctx_init_pd_0 },
+        { nvc0_grctx_init_rstr2d_0 },
+        { nvc0_grctx_init_scc_0 },
+        { nvd9_grctx_init_be_0 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvd9_grctx_init_prop_0[] = {
         { 0x418400,   1, 0x04, 0x38004e00 },
         { 0x418404,   1, 0x04, 0x71e0ffff },
         { 0x41840c,   1, 0x04, 0x00001008 },
@@ -368,11 +392,21 @@ nvd9_grctx_init_gpc_0[] = {
         { 0x418450,   6, 0x04, 0x00000000 },
         { 0x418468,   1, 0x04, 0x00000001 },
         { 0x41846c,   2, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvd9_grctx_init_gpc_unk_1[] = {
         { 0x418600,   1, 0x04, 0x0000001f },
         { 0x418684,   1, 0x04, 0x0000000f },
         { 0x418700,   1, 0x04, 0x00000002 },
         { 0x418704,   1, 0x04, 0x00000080 },
         { 0x418708,   3, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvd9_grctx_init_setup_0[] = {
         { 0x418800,   1, 0x04, 0x7006860a },
         { 0x418808,   3, 0x04, 0x00000000 },
         { 0x418828,   1, 0x04, 0x00008442 },
@@ -381,10 +415,11 @@ nvd9_grctx_init_gpc_0[] = {
         { 0x4188e0,   1, 0x04, 0x01000000 },
         { 0x4188e8,   5, 0x04, 0x00000000 },
         { 0x4188fc,   1, 0x04, 0x20100008 },
-        { 0x41891c,   1, 0x04, 0x00ff00ff },
-        { 0x418924,   1, 0x04, 0x00000000 },
-        { 0x418928,   1, 0x04, 0x00ffff00 },
-        { 0x41892c,   1, 0x04, 0x0000ff00 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvd9_grctx_init_crstr_0[] = {
         { 0x418b00,   1, 0x04, 0x00000006 },
         { 0x418b08,   1, 0x04, 0x0a418820 },
         { 0x418b0c,   1, 0x04, 0x062080e6 },
@@ -393,24 +428,24 @@ nvd9_grctx_init_gpc_0[] = {
         { 0x418b18,   1, 0x04, 0x0a418820 },
         { 0x418b1c,   1, 0x04, 0x000000e6 },
         { 0x418bb8,   1, 0x04, 0x00000103 },
-        { 0x418c08,   1, 0x04, 0x00000001 },
-        { 0x418c10,   8, 0x04, 0x00000000 },
-        { 0x418c6c,   1, 0x04, 0x00000001 },
-        { 0x418c80,   1, 0x04, 0x20200004 },
-        { 0x418c8c,   1, 0x04, 0x00000001 },
-        { 0x419000,   1, 0x04, 0x00000780 },
-        { 0x419004,   2, 0x04, 0x00000000 },
-        { 0x419014,   1, 0x04, 0x00000004 },
         {}
 };
 
-static struct nvc0_graph_init
-nvd9_grctx_init_tpc[] = {
-        { 0x419818,   1, 0x04, 0x00000000 },
-        { 0x41983c,   1, 0x04, 0x00038bc7 },
-        { 0x419848,   1, 0x04, 0x00000000 },
-        { 0x419864,   1, 0x04, 0x00000129 },
-        { 0x419888,   1, 0x04, 0x00000000 },
+static const struct nvc0_graph_pack
+nvd9_grctx_pack_gpc[] = {
+        { nvc0_grctx_init_gpc_unk_0 },
+        { nvd9_grctx_init_prop_0 },
+        { nvd9_grctx_init_gpc_unk_1 },
+        { nvd9_grctx_init_setup_0 },
+        { nvc0_grctx_init_zcull_0 },
+        { nvd9_grctx_init_crstr_0 },
+        { nvc1_grctx_init_gpm_0 },
+        { nvc0_grctx_init_gcc_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvd9_grctx_init_tex_0[] = {
         { 0x419a00,   1, 0x04, 0x000001f0 },
         { 0x419a04,   1, 0x04, 0x00000001 },
         { 0x419a08,   1, 0x04, 0x00000023 },
@@ -420,27 +455,22 @@ nvd9_grctx_init_tpc[] = {
         { 0x419a1c,   1, 0x04, 0x00000000 },
         { 0x419a20,   1, 0x04, 0x00000800 },
         { 0x419ac4,   1, 0x04, 0x0017f440 },
-        { 0x419b00,   1, 0x04, 0x0a418820 },
-        { 0x419b04,   1, 0x04, 0x062080e6 },
-        { 0x419b08,   1, 0x04, 0x020398a4 },
-        { 0x419b0c,   1, 0x04, 0x0e629062 },
-        { 0x419b10,   1, 0x04, 0x0a418820 },
-        { 0x419b14,   1, 0x04, 0x000000e6 },
-        { 0x419bd0,   1, 0x04, 0x00900103 },
-        { 0x419be0,   1, 0x04, 0x00400001 },
-        { 0x419be4,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvd9_grctx_init_mpc_0[] = {
         { 0x419c00,   1, 0x04, 0x0000000a },
         { 0x419c04,   1, 0x04, 0x00000006 },
         { 0x419c08,   1, 0x04, 0x00000002 },
         { 0x419c20,   1, 0x04, 0x00000000 },
         { 0x419c24,   1, 0x04, 0x00084210 },
         { 0x419c28,   1, 0x04, 0x3cf3cf3c },
-        { 0x419cb0,   1, 0x04, 0x00020048 },
-        { 0x419ce8,   1, 0x04, 0x00000000 },
-        { 0x419cf4,   1, 0x04, 0x00000183 },
-        { 0x419d20,   1, 0x04, 0x12180000 },
-        { 0x419d24,   1, 0x04, 0x00001fff },
-        { 0x419d44,   1, 0x04, 0x02180218 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvd9_grctx_init_sm_0[] = {
         { 0x419e04,   3, 0x04, 0x00000000 },
         { 0x419e10,   1, 0x04, 0x00000002 },
         { 0x419e44,   1, 0x04, 0x001beff2 },
@@ -453,47 +483,21 @@ nvd9_grctx_init_tpc[] = {
         {}
 };
 
-static struct nvc0_graph_init *
-nvd9_grctx_init_hub[] = {
-        nvc0_grctx_init_base,
-        nvd9_grctx_init_unk40xx,
-        nvc0_grctx_init_unk44xx,
-        nvc0_grctx_init_unk46xx,
-        nvc0_grctx_init_unk47xx,
-        nvd9_grctx_init_unk58xx,
-        nvc0_grctx_init_unk60xx,
-        nvd9_grctx_init_unk64xx,
-        nvc0_grctx_init_unk78xx,
-        nvc0_grctx_init_unk80xx,
-        nvd9_grctx_init_rop,
-        NULL
-};
-
-struct nvc0_graph_init *
-nvd9_grctx_init_gpc[] = {
-        nvd9_grctx_init_gpc_0,
-        nvc0_grctx_init_gpc_1,
-        nvd9_grctx_init_tpc,
-        NULL
-};
-
-struct nvc0_graph_init
-nvd9_grctx_init_mthd_magic[] = {
-        { 0x3410, 1, 0x04, 0x80002006 },
+static const struct nvc0_graph_pack
+nvd9_grctx_pack_tpc[] = {
+        { nvc1_grctx_init_pe_0 },
+        { nvd9_grctx_init_tex_0 },
+        { nvc1_grctx_init_wwdx_0 },
+        { nvd9_grctx_init_mpc_0 },
+        { nvc4_grctx_init_l1c_0 },
+        { nvc1_grctx_init_tpccs_0 },
+        { nvd9_grctx_init_sm_0 },
         {}
 };
 
-struct nvc0_graph_mthd
-nvd9_grctx_init_mthd[] = {
-        { 0x9097, nvc1_grctx_init_9097, },
-        { 0x9197, nvc8_grctx_init_9197, },
-        { 0x9297, nvc8_grctx_init_9297, },
-        { 0x902d, nvc0_grctx_init_902d, },
-        { 0x9039, nvc0_grctx_init_9039, },
-        { 0x90c0, nvd9_grctx_init_90c0, },
-        { 0x902d, nvd9_grctx_init_mthd_magic, },
-        {}
-};
+/*******************************************************************************
+ * PGRAPH context implementation
+ ******************************************************************************/
 
 struct nouveau_oclass *
 nvd9_grctx_oclass = &(struct nvc0_grctx_oclass) {
@@ -506,11 +510,13 @@ nvd9_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .rd32 = _nouveau_graph_context_rd32,
                 .wr32 = _nouveau_graph_context_wr32,
         },
-        .main = nvc0_grctx_generate_main,
-        .mods = nvc1_grctx_generate_mods,
-        .unkn = nvc1_grctx_generate_unkn,
-        .hub  = nvd9_grctx_init_hub,
-        .gpc  = nvd9_grctx_init_gpc,
-        .icmd = nvd9_grctx_init_icmd,
-        .mthd = nvd9_grctx_init_mthd,
+        .main  = nvc0_grctx_generate_main,
+        .mods  = nvc1_grctx_generate_mods,
+        .unkn  = nvc1_grctx_generate_unkn,
+        .hub   = nvd9_grctx_pack_hub,
+        .gpc   = nvd9_grctx_pack_gpc,
+        .zcull = nvc0_grctx_pack_zcull,
+        .tpc   = nvd9_grctx_pack_tpc,
+        .icmd  = nvd9_grctx_pack_icmd,
+        .mthd  = nvd9_grctx_pack_mthd,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnve4.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnve4.c
index e2de73ee5eee..49a14b116a5f 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnve4.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnve4.c
@@ -22,10 +22,14 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
-#include "nvc0.h"
+#include "ctxnvc0.h"
 
-struct nvc0_graph_init
-nve4_grctx_init_icmd[] = {
+/*******************************************************************************
+ * PGRAPH context register lists
+ ******************************************************************************/
+
+static const struct nvc0_graph_init
+nve4_grctx_init_icmd_0[] = {
         { 0x001000,   1, 0x01, 0x00000004 },
         { 0x000039,   3, 0x01, 0x00000000 },
         { 0x0000a9,   1, 0x01, 0x0000ffff },
@@ -138,8 +142,7 @@ nve4_grctx_init_icmd[] = {
         { 0x000586,   1, 0x01, 0x00000040 },
         { 0x000582,   2, 0x01, 0x00000080 },
         { 0x0005c2,   1, 0x01, 0x00000001 },
-        { 0x000638,   1, 0x01, 0x00000001 },
-        { 0x000639,   1, 0x01, 0x00000001 },
+        { 0x000638,   2, 0x01, 0x00000001 },
         { 0x00063a,   1, 0x01, 0x00000002 },
         { 0x00063b,   2, 0x01, 0x00000001 },
         { 0x00063d,   1, 0x01, 0x00000002 },
@@ -197,15 +200,13 @@ nve4_grctx_init_icmd[] = {
         { 0x000787,   1, 0x01, 0x000000cf },
         { 0x00078c,   1, 0x01, 0x00000008 },
         { 0x000792,   1, 0x01, 0x00000001 },
-        { 0x000794,   1, 0x01, 0x00000001 },
-        { 0x000795,   2, 0x01, 0x00000001 },
+        { 0x000794,   3, 0x01, 0x00000001 },
         { 0x000797,   1, 0x01, 0x000000cf },
         { 0x000836,   1, 0x01, 0x00000001 },
         { 0x00079a,   1, 0x01, 0x00000002 },
         { 0x000833,   1, 0x01, 0x04444480 },
         { 0x0007a1,   1, 0x01, 0x00000001 },
-        { 0x0007a3,   1, 0x01, 0x00000001 },
-        { 0x0007a4,   2, 0x01, 0x00000001 },
+        { 0x0007a3,   3, 0x01, 0x00000001 },
         { 0x000831,   1, 0x01, 0x00000004 },
         { 0x000b07,   1, 0x01, 0x00000002 },
         { 0x000b08,   2, 0x01, 0x00000100 },
@@ -231,14 +232,12 @@ nve4_grctx_init_icmd[] = {
         { 0x0006b1,   1, 0x01, 0x00000011 },
         { 0x00078c,   1, 0x01, 0x00000008 },
         { 0x000792,   1, 0x01, 0x00000001 },
-        { 0x000794,   1, 0x01, 0x00000001 },
-        { 0x000795,   2, 0x01, 0x00000001 },
+        { 0x000794,   3, 0x01, 0x00000001 },
         { 0x000797,   1, 0x01, 0x000000cf },
         { 0x00079a,   1, 0x01, 0x00000002 },
         { 0x000833,   1, 0x01, 0x04444480 },
         { 0x0007a1,   1, 0x01, 0x00000001 },
-        { 0x0007a3,   1, 0x01, 0x00000001 },
-        { 0x0007a4,   2, 0x01, 0x00000001 },
+        { 0x0007a3,   3, 0x01, 0x00000001 },
         { 0x000831,   1, 0x01, 0x00000004 },
         { 0x01e100,   1, 0x01, 0x00000001 },
         { 0x001000,   1, 0x01, 0x00000008 },
@@ -273,8 +272,14 @@ nve4_grctx_init_icmd[] = {
         {}
 };
 
-struct nvc0_graph_init
-nve4_grctx_init_a097[] = {
+static const struct nvc0_graph_pack
+nve4_grctx_pack_icmd[] = {
+        { nve4_grctx_init_icmd_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nve4_grctx_init_a097_0[] = {
         { 0x000800,   8, 0x40, 0x00000000 },
         { 0x000804,   8, 0x40, 0x00000000 },
         { 0x000808,   8, 0x40, 0x00000400 },
@@ -517,8 +522,7 @@ nve4_grctx_init_a097[] = {
         { 0x001350,   1, 0x04, 0x00000002 },
         { 0x001358,   1, 0x04, 0x00000001 },
         { 0x0012e4,   1, 0x04, 0x00000000 },
-        { 0x00131c,   1, 0x04, 0x00000000 },
-        { 0x001320,   3, 0x04, 0x00000000 },
+        { 0x00131c,   4, 0x04, 0x00000000 },
         { 0x0019c0,   1, 0x04, 0x00000000 },
         { 0x001140,   1, 0x04, 0x00000000 },
         { 0x0019c4,   1, 0x04, 0x00000000 },
@@ -574,19 +578,24 @@ nve4_grctx_init_a097[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nve4_grctx_init_unk40xx[] = {
+static const struct nvc0_graph_pack
+nve4_grctx_pack_mthd[] = {
+        { nve4_grctx_init_a097_0, 0xa097 },
+        { nvc0_grctx_init_902d_0, 0x902d },
+        {}
+};
+
+static const struct nvc0_graph_init
+nve4_grctx_init_fe_0[] = {
         { 0x404010,   5, 0x04, 0x00000000 },
         { 0x404024,   1, 0x04, 0x0000e000 },
         { 0x404028,   1, 0x04, 0x00000000 },
-        { 0x4040a8,   1, 0x04, 0x00000000 },
-        { 0x4040ac,   7, 0x04, 0x00000000 },
+        { 0x4040a8,   8, 0x04, 0x00000000 },
         { 0x4040c8,   1, 0x04, 0xf800008f },
         { 0x4040d0,   6, 0x04, 0x00000000 },
         { 0x4040e8,   1, 0x04, 0x00001000 },
         { 0x4040f8,   1, 0x04, 0x00000000 },
-        { 0x404130,   1, 0x04, 0x00000000 },
-        { 0x404134,   1, 0x04, 0x00000000 },
+        { 0x404130,   2, 0x04, 0x00000000 },
         { 0x404138,   1, 0x04, 0x20000040 },
         { 0x404150,   1, 0x04, 0x0000002e },
         { 0x404154,   1, 0x04, 0x00000400 },
@@ -597,8 +606,8 @@ nve4_grctx_init_unk40xx[] = {
         {}
 };
 
-struct nvc0_graph_init
-nve4_grctx_init_unk46xx[] = {
+const struct nvc0_graph_init
+nve4_grctx_init_memfmt_0[] = {
         { 0x404604,   1, 0x04, 0x00000014 },
         { 0x404608,   1, 0x04, 0x00000000 },
         { 0x40460c,   1, 0x04, 0x00003fff },
@@ -614,11 +623,6 @@ nve4_grctx_init_unk46xx[] = {
         { 0x4046a0,   1, 0x04, 0x007f0080 },
         { 0x4046a4,   8, 0x04, 0x00000000 },
         { 0x4046c8,   3, 0x04, 0x00000000 },
-        {}
-};
-
-struct nvc0_graph_init
-nve4_grctx_init_unk47xx[] = {
         { 0x404700,   3, 0x04, 0x00000000 },
         { 0x404718,   7, 0x04, 0x00000000 },
         { 0x404734,   1, 0x04, 0x00000100 },
@@ -628,8 +632,8 @@ nve4_grctx_init_unk47xx[] = {
         {}
 };
 
-struct nvc0_graph_init
-nve4_grctx_init_unk58xx[] = {
+const struct nvc0_graph_init
+nve4_grctx_init_ds_0[] = {
         { 0x405800,   1, 0x04, 0x0f8000bf },
         { 0x405830,   1, 0x04, 0x02180648 },
         { 0x405834,   1, 0x04, 0x08000000 },
@@ -641,22 +645,17 @@ nve4_grctx_init_unk58xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nve4_grctx_init_unk5bxx[] = {
+static const struct nvc0_graph_init
+nve4_grctx_init_cwd_0[] = {
         { 0x405b00,   1, 0x04, 0x00000000 },
         { 0x405b10,   1, 0x04, 0x00001000 },
         {}
 };
 
-static struct nvc0_graph_init
-nve4_grctx_init_unk60xx[] = {
+static const struct nvc0_graph_init
+nve4_grctx_init_pd_0[] = {
         { 0x406020,   1, 0x04, 0x004103c1 },
         { 0x406028,   4, 0x04, 0x00000001 },
-        {}
-};
-
-static struct nvc0_graph_init
-nve4_grctx_init_unk64xx[] = {
         { 0x4064a8,   1, 0x04, 0x00000000 },
         { 0x4064ac,   1, 0x04, 0x00003fff },
         { 0x4064b4,   2, 0x04, 0x00000000 },
@@ -668,14 +667,14 @@ nve4_grctx_init_unk64xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nve4_grctx_init_unk70xx[] = {
+static const struct nvc0_graph_init
+nve4_grctx_init_sked_0[] = {
         { 0x407040,   1, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nve4_grctx_init_unk80xx[] = {
+const struct nvc0_graph_init
+nve4_grctx_init_scc_0[] = {
         { 0x408000,   2, 0x04, 0x00000000 },
         { 0x408008,   1, 0x04, 0x00000030 },
         { 0x40800c,   2, 0x04, 0x00000000 },
@@ -685,8 +684,8 @@ nve4_grctx_init_unk80xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nve4_grctx_init_rop[] = {
+static const struct nvc0_graph_init
+nve4_grctx_init_be_0[] = {
         { 0x408800,   1, 0x04, 0x02802a3c },
         { 0x408804,   1, 0x04, 0x00000040 },
         { 0x408808,   1, 0x04, 0x1043e005 },
@@ -698,22 +697,24 @@ nve4_grctx_init_rop[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nve4_grctx_init_gpc_0[] = {
-        { 0x418380,   1, 0x04, 0x00000016 },
-        { 0x418400,   1, 0x04, 0x38004e00 },
-        { 0x418404,   1, 0x04, 0x71e0ffff },
-        { 0x41840c,   1, 0x04, 0x00001008 },
-        { 0x418410,   1, 0x04, 0x0fff0fff },
-        { 0x418414,   1, 0x04, 0x02200fff },
-        { 0x418450,   6, 0x04, 0x00000000 },
-        { 0x418468,   1, 0x04, 0x00000001 },
-        { 0x41846c,   2, 0x04, 0x00000000 },
-        { 0x418600,   1, 0x04, 0x0000001f },
-        { 0x418684,   1, 0x04, 0x0000000f },
-        { 0x418700,   1, 0x04, 0x00000002 },
-        { 0x418704,   1, 0x04, 0x00000080 },
-        { 0x418708,   3, 0x04, 0x00000000 },
+static const struct nvc0_graph_pack
+nve4_grctx_pack_hub[] = {
+        { nvc0_grctx_init_main_0 },
+        { nve4_grctx_init_fe_0 },
+        { nvc0_grctx_init_pri_0 },
+        { nve4_grctx_init_memfmt_0 },
+        { nve4_grctx_init_ds_0 },
+        { nve4_grctx_init_cwd_0 },
+        { nve4_grctx_init_pd_0 },
+        { nve4_grctx_init_sked_0 },
+        { nvc0_grctx_init_rstr2d_0 },
+        { nve4_grctx_init_scc_0 },
+        { nve4_grctx_init_be_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nve4_grctx_init_setup_0[] = {
         { 0x418800,   1, 0x04, 0x7006860a },
         { 0x418808,   3, 0x04, 0x00000000 },
         { 0x418828,   1, 0x04, 0x00000044 },
@@ -722,35 +723,35 @@ nve4_grctx_init_gpc_0[] = {
         { 0x4188e0,   1, 0x04, 0x01000000 },
         { 0x4188e8,   5, 0x04, 0x00000000 },
         { 0x4188fc,   1, 0x04, 0x20100018 },
-        { 0x41891c,   1, 0x04, 0x00ff00ff },
-        { 0x418924,   1, 0x04, 0x00000000 },
-        { 0x418928,   1, 0x04, 0x00ffff00 },
-        { 0x41892c,   1, 0x04, 0x0000ff00 },
-        { 0x418b00,   1, 0x04, 0x00000006 },
-        { 0x418b08,   1, 0x04, 0x0a418820 },
-        { 0x418b0c,   1, 0x04, 0x062080e6 },
-        { 0x418b10,   1, 0x04, 0x020398a4 },
-        { 0x418b14,   1, 0x04, 0x0e629062 },
-        { 0x418b18,   1, 0x04, 0x0a418820 },
-        { 0x418b1c,   1, 0x04, 0x000000e6 },
-        { 0x418bb8,   1, 0x04, 0x00000103 },
+        {}
+};
+
+const struct nvc0_graph_init
+nve4_grctx_init_gpm_0[] = {
         { 0x418c08,   1, 0x04, 0x00000001 },
         { 0x418c10,   8, 0x04, 0x00000000 },
         { 0x418c40,   1, 0x04, 0xffffffff },
         { 0x418c6c,   1, 0x04, 0x00000001 },
         { 0x418c80,   1, 0x04, 0x20200004 },
         { 0x418c8c,   1, 0x04, 0x00000001 },
-        { 0x419000,   1, 0x04, 0x00000780 },
-        { 0x419004,   2, 0x04, 0x00000000 },
-        { 0x419014,   1, 0x04, 0x00000004 },
         {}
 };
 
-static struct nvc0_graph_init
-nve4_grctx_init_tpc[] = {
-        { 0x419848,   1, 0x04, 0x00000000 },
-        { 0x419864,   1, 0x04, 0x00000129 },
-        { 0x419888,   1, 0x04, 0x00000000 },
+static const struct nvc0_graph_pack
+nve4_grctx_pack_gpc[] = {
+        { nvc0_grctx_init_gpc_unk_0 },
+        { nvd9_grctx_init_prop_0 },
+        { nvd9_grctx_init_gpc_unk_1 },
+        { nve4_grctx_init_setup_0 },
+        { nvc0_grctx_init_zcull_0 },
+        { nvd9_grctx_init_crstr_0 },
+        { nve4_grctx_init_gpm_0 },
+        { nvc0_grctx_init_gcc_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nve4_grctx_init_tex_0[] = {
         { 0x419a00,   1, 0x04, 0x000000f0 },
         { 0x419a04,   1, 0x04, 0x00000001 },
         { 0x419a08,   1, 0x04, 0x00000021 },
@@ -761,14 +762,29 @@ nve4_grctx_init_tpc[] = {
         { 0x419a20,   1, 0x04, 0x00000800 },
         { 0x419a30,   1, 0x04, 0x00000001 },
         { 0x419ac4,   1, 0x04, 0x0037f440 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nve4_grctx_init_mpc_0[] = {
         { 0x419c00,   1, 0x04, 0x0000000a },
         { 0x419c04,   1, 0x04, 0x80000006 },
         { 0x419c08,   1, 0x04, 0x00000002 },
         { 0x419c20,   1, 0x04, 0x00000000 },
         { 0x419c24,   1, 0x04, 0x00084210 },
         { 0x419c28,   1, 0x04, 0x3efbefbe },
+        {}
+};
+
+static const struct nvc0_graph_init
+nve4_grctx_init_l1c_0[] = {
         { 0x419ce8,   1, 0x04, 0x00000000 },
         { 0x419cf4,   1, 0x04, 0x00003203 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nve4_grctx_init_sm_0[] = {
         { 0x419e04,   3, 0x04, 0x00000000 },
         { 0x419e10,   1, 0x04, 0x00000402 },
         { 0x419e44,   1, 0x04, 0x0013eff2 },
@@ -782,28 +798,46 @@ nve4_grctx_init_tpc[] = {
         { 0x419f58,   1, 0x04, 0x00000000 },
         { 0x419f70,   1, 0x04, 0x00000000 },
         { 0x419f78,   1, 0x04, 0x0000000b },
-        { 0x419f7c,   1, 0x04, 0x0000027a },
+        { 0x419f7c,   1, 0x04, 0x0000027c },
+        {}
+};
+
+static const struct nvc0_graph_pack
+nve4_grctx_pack_tpc[] = {
+        { nvd7_grctx_init_pe_0 },
+        { nve4_grctx_init_tex_0 },
+        { nve4_grctx_init_mpc_0 },
+        { nve4_grctx_init_l1c_0 },
+        { nve4_grctx_init_sm_0 },
         {}
 };
 
-static struct nvc0_graph_init
-nve4_grctx_init_unk[] = {
+const struct nvc0_graph_init
+nve4_grctx_init_pes_0[] = {
         { 0x41be24,   1, 0x04, 0x00000006 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nve4_grctx_init_cbm_0[] = {
         { 0x41bec0,   1, 0x04, 0x12180000 },
         { 0x41bec4,   1, 0x04, 0x00037f7f },
         { 0x41bee4,   1, 0x04, 0x06480430 },
-        { 0x41bf00,   1, 0x04, 0x0a418820 },
-        { 0x41bf04,   1, 0x04, 0x062080e6 },
-        { 0x41bf08,   1, 0x04, 0x020398a4 },
-        { 0x41bf0c,   1, 0x04, 0x0e629062 },
-        { 0x41bf10,   1, 0x04, 0x0a418820 },
-        { 0x41bf14,   1, 0x04, 0x000000e6 },
-        { 0x41bfd0,   1, 0x04, 0x00900103 },
-        { 0x41bfe0,   1, 0x04, 0x00400001 },
-        { 0x41bfe4,   1, 0x04, 0x00000000 },
         {}
 };
 
+static const struct nvc0_graph_pack
+nve4_grctx_pack_ppc[] = {
+        { nve4_grctx_init_pes_0 },
+        { nve4_grctx_init_cbm_0 },
+        { nvd7_grctx_init_wwdx_0 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH context implementation
+ ******************************************************************************/
+
 static void
 nve4_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
 {
@@ -925,10 +959,11 @@ nve4_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
 
         nv_mask(priv, 0x000260, 0x00000001, 0x00000000);
 
-        for (i = 0; oclass->hub[i]; i++)
-                nvc0_graph_mmio(priv, oclass->hub[i]);
-        for (i = 0; oclass->gpc[i]; i++)
-                nvc0_graph_mmio(priv, oclass->gpc[i]);
+        nvc0_graph_mmio(priv, oclass->hub);
+        nvc0_graph_mmio(priv, oclass->gpc);
+        nvc0_graph_mmio(priv, oclass->zcull);
+        nvc0_graph_mmio(priv, oclass->tpc);
+        nvc0_graph_mmio(priv, oclass->ppc);
 
         nv_wr32(priv, 0x404154, 0x00000000);
 
@@ -962,41 +997,6 @@ nve4_grctx_generate_main(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
         nv_mask(priv, 0x41be10, 0x00800000, 0x00800000);
 }
 
-static struct nvc0_graph_init *
-nve4_grctx_init_hub[] = {
-        nvc0_grctx_init_base,
-        nve4_grctx_init_unk40xx,
-        nvc0_grctx_init_unk44xx,
-        nve4_grctx_init_unk46xx,
-        nve4_grctx_init_unk47xx,
-        nve4_grctx_init_unk58xx,
-        nve4_grctx_init_unk5bxx,
-        nve4_grctx_init_unk60xx,
-        nve4_grctx_init_unk64xx,
-        nve4_grctx_init_unk70xx,
-        nvc0_grctx_init_unk78xx,
-        nve4_grctx_init_unk80xx,
-        nve4_grctx_init_rop,
-        NULL
-};
-
-struct nvc0_graph_init *
-nve4_grctx_init_gpc[] = {
-        nve4_grctx_init_gpc_0,
-        nvc0_grctx_init_gpc_1,
-        nve4_grctx_init_tpc,
-        nve4_grctx_init_unk,
-        NULL
-};
-
-static struct nvc0_graph_mthd
-nve4_grctx_init_mthd[] = {
-        { 0xa097, nve4_grctx_init_a097, },
-        { 0x902d, nvc0_grctx_init_902d, },
-        { 0x902d, nvc0_grctx_init_mthd_magic, },
-        {}
-};
-
 struct nouveau_oclass *
 nve4_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .base.handle = NV_ENGCTX(GR, 0xe4),
@@ -1008,11 +1008,14 @@ nve4_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .rd32 = _nouveau_graph_context_rd32,
                 .wr32 = _nouveau_graph_context_wr32,
         },
-        .main = nve4_grctx_generate_main,
-        .mods = nve4_grctx_generate_mods,
-        .unkn = nve4_grctx_generate_unkn,
-        .hub  = nve4_grctx_init_hub,
-        .gpc  = nve4_grctx_init_gpc,
-        .icmd = nve4_grctx_init_icmd,
-        .mthd = nve4_grctx_init_mthd,
+        .main  = nve4_grctx_generate_main,
+        .mods  = nve4_grctx_generate_mods,
+        .unkn  = nve4_grctx_generate_unkn,
+        .hub   = nve4_grctx_pack_hub,
+        .gpc   = nve4_grctx_pack_gpc,
+        .zcull = nvc0_grctx_pack_zcull,
+        .tpc   = nve4_grctx_pack_tpc,
+        .ppc   = nve4_grctx_pack_ppc,
+        .icmd  = nve4_grctx_pack_icmd,
+        .mthd  = nve4_grctx_pack_mthd,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvf0.c b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvf0.c
index 44012c3da538..0fab95e49f53 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvf0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/ctxnvf0.c
@@ -22,10 +22,580 @@
  * Authors: Ben Skeggs <bskeggs@redhat.com>
  */
 
-#include "nvc0.h"
+#include "ctxnvc0.h"
 
-static struct nvc0_graph_init
-nvf0_grctx_init_unk40xx[] = {
+/*******************************************************************************
+ * PGRAPH context register lists
+ ******************************************************************************/
+
+static const struct nvc0_graph_init
+nvf0_grctx_init_icmd_0[] = {
+        { 0x001000,   1, 0x01, 0x00000004 },
+        { 0x000039,   3, 0x01, 0x00000000 },
+        { 0x0000a9,   1, 0x01, 0x0000ffff },
+        { 0x000038,   1, 0x01, 0x0fac6881 },
+        { 0x00003d,   1, 0x01, 0x00000001 },
+        { 0x0000e8,   8, 0x01, 0x00000400 },
+        { 0x000078,   8, 0x01, 0x00000300 },
+        { 0x000050,   1, 0x01, 0x00000011 },
+        { 0x000058,   8, 0x01, 0x00000008 },
+        { 0x000208,   8, 0x01, 0x00000001 },
+        { 0x000081,   1, 0x01, 0x00000001 },
+        { 0x000085,   1, 0x01, 0x00000004 },
+        { 0x000088,   1, 0x01, 0x00000400 },
+        { 0x000090,   1, 0x01, 0x00000300 },
+        { 0x000098,   1, 0x01, 0x00001001 },
+        { 0x0000e3,   1, 0x01, 0x00000001 },
+        { 0x0000da,   1, 0x01, 0x00000001 },
+        { 0x0000f8,   1, 0x01, 0x00000003 },
+        { 0x0000fa,   1, 0x01, 0x00000001 },
+        { 0x00009f,   4, 0x01, 0x0000ffff },
+        { 0x0000b1,   1, 0x01, 0x00000001 },
+        { 0x0000ad,   1, 0x01, 0x0000013e },
+        { 0x0000e1,   1, 0x01, 0x00000010 },
+        { 0x000290,  16, 0x01, 0x00000000 },
+        { 0x0003b0,  16, 0x01, 0x00000000 },
+        { 0x0002a0,  16, 0x01, 0x00000000 },
+        { 0x000420,  16, 0x01, 0x00000000 },
+        { 0x0002b0,  16, 0x01, 0x00000000 },
+        { 0x000430,  16, 0x01, 0x00000000 },
+        { 0x0002c0,  16, 0x01, 0x00000000 },
+        { 0x0004d0,  16, 0x01, 0x00000000 },
+        { 0x000720,  16, 0x01, 0x00000000 },
+        { 0x0008c0,  16, 0x01, 0x00000000 },
+        { 0x000890,  16, 0x01, 0x00000000 },
+        { 0x0008e0,  16, 0x01, 0x00000000 },
+        { 0x0008a0,  16, 0x01, 0x00000000 },
+        { 0x0008f0,  16, 0x01, 0x00000000 },
+        { 0x00094c,   1, 0x01, 0x000000ff },
+        { 0x00094d,   1, 0x01, 0xffffffff },
+        { 0x00094e,   1, 0x01, 0x00000002 },
+        { 0x0002ec,   1, 0x01, 0x00000001 },
+        { 0x0002f2,   2, 0x01, 0x00000001 },
+        { 0x0002f5,   1, 0x01, 0x00000001 },
+        { 0x0002f7,   1, 0x01, 0x00000001 },
+        { 0x000303,   1, 0x01, 0x00000001 },
+        { 0x0002e6,   1, 0x01, 0x00000001 },
+        { 0x000466,   1, 0x01, 0x00000052 },
+        { 0x000301,   1, 0x01, 0x3f800000 },
+        { 0x000304,   1, 0x01, 0x30201000 },
+        { 0x000305,   1, 0x01, 0x70605040 },
+        { 0x000306,   1, 0x01, 0xb8a89888 },
+        { 0x000307,   1, 0x01, 0xf8e8d8c8 },
+        { 0x00030a,   1, 0x01, 0x00ffff00 },
+        { 0x00030b,   1, 0x01, 0x0000001a },
+        { 0x00030c,   1, 0x01, 0x00000001 },
+        { 0x000318,   1, 0x01, 0x00000001 },
+        { 0x000340,   1, 0x01, 0x00000000 },
+        { 0x000375,   1, 0x01, 0x00000001 },
+        { 0x00037d,   1, 0x01, 0x00000006 },
+        { 0x0003a0,   1, 0x01, 0x00000002 },
+        { 0x0003aa,   1, 0x01, 0x00000001 },
+        { 0x0003a9,   1, 0x01, 0x00000001 },
+        { 0x000380,   1, 0x01, 0x00000001 },
+        { 0x000383,   1, 0x01, 0x00000011 },
+        { 0x000360,   1, 0x01, 0x00000040 },
+        { 0x000366,   2, 0x01, 0x00000000 },
+        { 0x000368,   1, 0x01, 0x00000fff },
+        { 0x000370,   2, 0x01, 0x00000000 },
+        { 0x000372,   1, 0x01, 0x000fffff },
+        { 0x00037a,   1, 0x01, 0x00000012 },
+        { 0x000619,   1, 0x01, 0x00000003 },
+        { 0x000811,   1, 0x01, 0x00000003 },
+        { 0x000812,   1, 0x01, 0x00000004 },
+        { 0x000813,   1, 0x01, 0x00000006 },
+        { 0x000814,   1, 0x01, 0x00000008 },
+        { 0x000815,   1, 0x01, 0x0000000b },
+        { 0x000800,   6, 0x01, 0x00000001 },
+        { 0x000632,   1, 0x01, 0x00000001 },
+        { 0x000633,   1, 0x01, 0x00000002 },
+        { 0x000634,   1, 0x01, 0x00000003 },
+        { 0x000635,   1, 0x01, 0x00000004 },
+        { 0x000654,   1, 0x01, 0x3f800000 },
+        { 0x000657,   1, 0x01, 0x3f800000 },
+        { 0x000655,   2, 0x01, 0x3f800000 },
+        { 0x0006cd,   1, 0x01, 0x3f800000 },
+        { 0x0007f5,   1, 0x01, 0x3f800000 },
+        { 0x0007dc,   1, 0x01, 0x39291909 },
+        { 0x0007dd,   1, 0x01, 0x79695949 },
+        { 0x0007de,   1, 0x01, 0xb9a99989 },
+        { 0x0007df,   1, 0x01, 0xf9e9d9c9 },
+        { 0x0007e8,   1, 0x01, 0x00003210 },
+        { 0x0007e9,   1, 0x01, 0x00007654 },
+        { 0x0007ea,   1, 0x01, 0x00000098 },
+        { 0x0007ec,   1, 0x01, 0x39291909 },
+        { 0x0007ed,   1, 0x01, 0x79695949 },
+        { 0x0007ee,   1, 0x01, 0xb9a99989 },
+        { 0x0007ef,   1, 0x01, 0xf9e9d9c9 },
+        { 0x0007f0,   1, 0x01, 0x00003210 },
+        { 0x0007f1,   1, 0x01, 0x00007654 },
+        { 0x0007f2,   1, 0x01, 0x00000098 },
+        { 0x0005a5,   1, 0x01, 0x00000001 },
+        { 0x000980, 128, 0x01, 0x00000000 },
+        { 0x000468,   1, 0x01, 0x00000004 },
+        { 0x00046c,   1, 0x01, 0x00000001 },
+        { 0x000470,  96, 0x01, 0x00000000 },
+        { 0x000510,  16, 0x01, 0x3f800000 },
+        { 0x000520,   1, 0x01, 0x000002b6 },
+        { 0x000529,   1, 0x01, 0x00000001 },
+        { 0x000530,  16, 0x01, 0xffff0000 },
+        { 0x000585,   1, 0x01, 0x0000003f },
+        { 0x000576,   1, 0x01, 0x00000003 },
+        { 0x00057b,   1, 0x01, 0x00000059 },
+        { 0x000586,   1, 0x01, 0x00000040 },
+        { 0x000582,   2, 0x01, 0x00000080 },
+        { 0x0005c2,   1, 0x01, 0x00000001 },
+        { 0x000638,   2, 0x01, 0x00000001 },
+        { 0x00063a,   1, 0x01, 0x00000002 },
+        { 0x00063b,   2, 0x01, 0x00000001 },
+        { 0x00063d,   1, 0x01, 0x00000002 },
+        { 0x00063e,   1, 0x01, 0x00000001 },
+        { 0x0008b8,   8, 0x01, 0x00000001 },
+        { 0x000900,   8, 0x01, 0x00000001 },
+        { 0x000908,   8, 0x01, 0x00000002 },
+        { 0x000910,  16, 0x01, 0x00000001 },
+        { 0x000920,   8, 0x01, 0x00000002 },
+        { 0x000928,   8, 0x01, 0x00000001 },
+        { 0x000662,   1, 0x01, 0x00000001 },
+        { 0x000648,   9, 0x01, 0x00000001 },
+        { 0x000658,   1, 0x01, 0x0000000f },
+        { 0x0007ff,   1, 0x01, 0x0000000a },
+        { 0x00066a,   1, 0x01, 0x40000000 },
+        { 0x00066b,   1, 0x01, 0x10000000 },
+        { 0x00066c,   2, 0x01, 0xffff0000 },
+        { 0x0007af,   2, 0x01, 0x00000008 },
+        { 0x0007f6,   1, 0x01, 0x00000001 },
+        { 0x00080b,   1, 0x01, 0x00000002 },
+        { 0x0006b2,   1, 0x01, 0x00000055 },
+        { 0x0007ad,   1, 0x01, 0x00000003 },
+        { 0x000937,   1, 0x01, 0x00000001 },
+        { 0x000971,   1, 0x01, 0x00000008 },
+        { 0x000972,   1, 0x01, 0x00000040 },
+        { 0x000973,   1, 0x01, 0x0000012c },
+        { 0x00097c,   1, 0x01, 0x00000040 },
+        { 0x000979,   1, 0x01, 0x00000003 },
+        { 0x000975,   1, 0x01, 0x00000020 },
+        { 0x000976,   1, 0x01, 0x00000001 },
+        { 0x000977,   1, 0x01, 0x00000020 },
+        { 0x000978,   1, 0x01, 0x00000001 },
+        { 0x000957,   1, 0x01, 0x00000003 },
+        { 0x00095e,   1, 0x01, 0x20164010 },
+        { 0x00095f,   1, 0x01, 0x00000020 },
+        { 0x000a0d,   1, 0x01, 0x00000006 },
+        { 0x00097d,   1, 0x01, 0x00000020 },
+        { 0x000683,   1, 0x01, 0x00000006 },
+        { 0x000685,   1, 0x01, 0x003fffff },
+        { 0x000687,   1, 0x01, 0x003fffff },
+        { 0x0006a0,   1, 0x01, 0x00000005 },
+        { 0x000840,   1, 0x01, 0x00400008 },
+        { 0x000841,   1, 0x01, 0x08000080 },
+        { 0x000842,   1, 0x01, 0x00400008 },
+        { 0x000843,   1, 0x01, 0x08000080 },
+        { 0x0006aa,   1, 0x01, 0x00000001 },
+        { 0x0006ab,   1, 0x01, 0x00000002 },
+        { 0x0006ac,   1, 0x01, 0x00000080 },
+        { 0x0006ad,   2, 0x01, 0x00000100 },
+        { 0x0006b1,   1, 0x01, 0x00000011 },
+        { 0x0006bb,   1, 0x01, 0x000000cf },
+        { 0x0006ce,   1, 0x01, 0x2a712488 },
+        { 0x000739,   1, 0x01, 0x4085c000 },
+        { 0x00073a,   1, 0x01, 0x00000080 },
+        { 0x000786,   1, 0x01, 0x80000100 },
+        { 0x00073c,   1, 0x01, 0x00010100 },
+        { 0x00073d,   1, 0x01, 0x02800000 },
+        { 0x000787,   1, 0x01, 0x000000cf },
+        { 0x00078c,   1, 0x01, 0x00000008 },
+        { 0x000792,   1, 0x01, 0x00000001 },
+        { 0x000794,   3, 0x01, 0x00000001 },
+        { 0x000797,   1, 0x01, 0x000000cf },
+        { 0x000836,   1, 0x01, 0x00000001 },
+        { 0x00079a,   1, 0x01, 0x00000002 },
+        { 0x000833,   1, 0x01, 0x04444480 },
+        { 0x0007a1,   1, 0x01, 0x00000001 },
+        { 0x0007a3,   3, 0x01, 0x00000001 },
+        { 0x000831,   1, 0x01, 0x00000004 },
+        { 0x000b07,   1, 0x01, 0x00000002 },
+        { 0x000b08,   2, 0x01, 0x00000100 },
+        { 0x000b0a,   1, 0x01, 0x00000001 },
+        { 0x000a04,   1, 0x01, 0x000000ff },
+        { 0x000a0b,   1, 0x01, 0x00000040 },
+        { 0x00097f,   1, 0x01, 0x00000100 },
+        { 0x000a02,   1, 0x01, 0x00000001 },
+        { 0x000809,   1, 0x01, 0x00000007 },
+        { 0x00c221,   1, 0x01, 0x00000040 },
+        { 0x00c1b0,   8, 0x01, 0x0000000f },
+        { 0x00c1b8,   1, 0x01, 0x0fac6881 },
+        { 0x00c1b9,   1, 0x01, 0x00fac688 },
+        { 0x00c401,   1, 0x01, 0x00000001 },
+        { 0x00c402,   1, 0x01, 0x00010001 },
+        { 0x00c403,   2, 0x01, 0x00000001 },
+        { 0x00c40e,   1, 0x01, 0x00000020 },
+        { 0x00c500,   1, 0x01, 0x00000003 },
+        { 0x01e100,   1, 0x01, 0x00000001 },
+        { 0x001000,   1, 0x01, 0x00000002 },
+        { 0x0006aa,   1, 0x01, 0x00000001 },
+        { 0x0006ad,   2, 0x01, 0x00000100 },
+        { 0x0006b1,   1, 0x01, 0x00000011 },
+        { 0x00078c,   1, 0x01, 0x00000008 },
+        { 0x000792,   1, 0x01, 0x00000001 },
+        { 0x000794,   3, 0x01, 0x00000001 },
+        { 0x000797,   1, 0x01, 0x000000cf },
+        { 0x00079a,   1, 0x01, 0x00000002 },
+        { 0x000833,   1, 0x01, 0x04444480 },
+        { 0x0007a1,   1, 0x01, 0x00000001 },
+        { 0x0007a3,   3, 0x01, 0x00000001 },
+        { 0x000831,   1, 0x01, 0x00000004 },
+        { 0x01e100,   1, 0x01, 0x00000001 },
+        { 0x001000,   1, 0x01, 0x00000008 },
+        { 0x000039,   3, 0x01, 0x00000000 },
+        { 0x000380,   1, 0x01, 0x00000001 },
+        { 0x000366,   2, 0x01, 0x00000000 },
+        { 0x000368,   1, 0x01, 0x00000fff },
+        { 0x000370,   2, 0x01, 0x00000000 },
+        { 0x000372,   1, 0x01, 0x000fffff },
+        { 0x000813,   1, 0x01, 0x00000006 },
+        { 0x000814,   1, 0x01, 0x00000008 },
+        { 0x000957,   1, 0x01, 0x00000003 },
+        { 0x000b07,   1, 0x01, 0x00000002 },
+        { 0x000b08,   2, 0x01, 0x00000100 },
+        { 0x000b0a,   1, 0x01, 0x00000001 },
+        { 0x000a04,   1, 0x01, 0x000000ff },
+        { 0x000a0b,   1, 0x01, 0x00000040 },
+        { 0x00097f,   1, 0x01, 0x00000100 },
+        { 0x000a02,   1, 0x01, 0x00000001 },
+        { 0x000809,   1, 0x01, 0x00000007 },
+        { 0x00c221,   1, 0x01, 0x00000040 },
+        { 0x00c401,   1, 0x01, 0x00000001 },
+        { 0x00c402,   1, 0x01, 0x00010001 },
+        { 0x00c403,   2, 0x01, 0x00000001 },
+        { 0x00c40e,   1, 0x01, 0x00000020 },
+        { 0x00c500,   1, 0x01, 0x00000003 },
+        { 0x01e100,   1, 0x01, 0x00000001 },
+        { 0x001000,   1, 0x01, 0x00000001 },
+        { 0x000b07,   1, 0x01, 0x00000002 },
+        { 0x000b08,   2, 0x01, 0x00000100 },
+        { 0x000b0a,   1, 0x01, 0x00000001 },
+        { 0x01e100,   1, 0x01, 0x00000001 },
+        {}
+};
+
+static const struct nvc0_graph_pack
+nvf0_grctx_pack_icmd[] = {
+        { nvf0_grctx_init_icmd_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvf0_grctx_init_a197_0[] = {
+        { 0x000800,   8, 0x40, 0x00000000 },
+        { 0x000804,   8, 0x40, 0x00000000 },
+        { 0x000808,   8, 0x40, 0x00000400 },
+        { 0x00080c,   8, 0x40, 0x00000300 },
+        { 0x000810,   1, 0x04, 0x000000cf },
+        { 0x000850,   7, 0x40, 0x00000000 },
+        { 0x000814,   8, 0x40, 0x00000040 },
+        { 0x000818,   8, 0x40, 0x00000001 },
+        { 0x00081c,   8, 0x40, 0x00000000 },
+        { 0x000820,   8, 0x40, 0x00000000 },
+        { 0x001c00,  16, 0x10, 0x00000000 },
+        { 0x001c04,  16, 0x10, 0x00000000 },
+        { 0x001c08,  16, 0x10, 0x00000000 },
+        { 0x001c0c,  16, 0x10, 0x00000000 },
+        { 0x001d00,  16, 0x10, 0x00000000 },
+        { 0x001d04,  16, 0x10, 0x00000000 },
+        { 0x001d08,  16, 0x10, 0x00000000 },
+        { 0x001d0c,  16, 0x10, 0x00000000 },
+        { 0x001f00,  16, 0x08, 0x00000000 },
+        { 0x001f04,  16, 0x08, 0x00000000 },
+        { 0x001f80,  16, 0x08, 0x00000000 },
+        { 0x001f84,  16, 0x08, 0x00000000 },
+        { 0x002000,   1, 0x04, 0x00000000 },
+        { 0x002040,   1, 0x04, 0x00000011 },
+        { 0x002080,   1, 0x04, 0x00000020 },
+        { 0x0020c0,   1, 0x04, 0x00000030 },
+        { 0x002100,   1, 0x04, 0x00000040 },
+        { 0x002140,   1, 0x04, 0x00000051 },
+        { 0x00200c,   6, 0x40, 0x00000001 },
+        { 0x002010,   1, 0x04, 0x00000000 },
+        { 0x002050,   1, 0x04, 0x00000000 },
+        { 0x002090,   1, 0x04, 0x00000001 },
+        { 0x0020d0,   1, 0x04, 0x00000002 },
+        { 0x002110,   1, 0x04, 0x00000003 },
+        { 0x002150,   1, 0x04, 0x00000004 },
+        { 0x000380,   4, 0x20, 0x00000000 },
+        { 0x000384,   4, 0x20, 0x00000000 },
+        { 0x000388,   4, 0x20, 0x00000000 },
+        { 0x00038c,   4, 0x20, 0x00000000 },
+        { 0x000700,   4, 0x10, 0x00000000 },
+        { 0x000704,   4, 0x10, 0x00000000 },
+        { 0x000708,   4, 0x10, 0x00000000 },
+        { 0x002800, 128, 0x04, 0x00000000 },
+        { 0x000a00,  16, 0x20, 0x00000000 },
+        { 0x000a04,  16, 0x20, 0x00000000 },
+        { 0x000a08,  16, 0x20, 0x00000000 },
+        { 0x000a0c,  16, 0x20, 0x00000000 },
+        { 0x000a10,  16, 0x20, 0x00000000 },
+        { 0x000a14,  16, 0x20, 0x00000000 },
+        { 0x000c00,  16, 0x10, 0x00000000 },
+        { 0x000c04,  16, 0x10, 0x00000000 },
+        { 0x000c08,  16, 0x10, 0x00000000 },
+        { 0x000c0c,  16, 0x10, 0x3f800000 },
+        { 0x000d00,   8, 0x08, 0xffff0000 },
+        { 0x000d04,   8, 0x08, 0xffff0000 },
+        { 0x000e00,  16, 0x10, 0x00000000 },
+        { 0x000e04,  16, 0x10, 0xffff0000 },
+        { 0x000e08,  16, 0x10, 0xffff0000 },
+        { 0x000d40,   4, 0x08, 0x00000000 },
+        { 0x000d44,   4, 0x08, 0x00000000 },
+        { 0x001e00,   8, 0x20, 0x00000001 },
+        { 0x001e04,   8, 0x20, 0x00000001 },
+        { 0x001e08,   8, 0x20, 0x00000002 },
+        { 0x001e0c,   8, 0x20, 0x00000001 },
+        { 0x001e10,   8, 0x20, 0x00000001 },
+        { 0x001e14,   8, 0x20, 0x00000002 },
+        { 0x001e18,   8, 0x20, 0x00000001 },
+        { 0x003400, 128, 0x04, 0x00000000 },
+        { 0x00030c,   1, 0x04, 0x00000001 },
+        { 0x001944,   1, 0x04, 0x00000000 },
+        { 0x001514,   1, 0x04, 0x00000000 },
+        { 0x000d68,   1, 0x04, 0x0000ffff },
+        { 0x00121c,   1, 0x04, 0x0fac6881 },
+        { 0x000fac,   1, 0x04, 0x00000001 },
+        { 0x001538,   1, 0x04, 0x00000001 },
+        { 0x000fe0,   2, 0x04, 0x00000000 },
+        { 0x000fe8,   1, 0x04, 0x00000014 },
+        { 0x000fec,   1, 0x04, 0x00000040 },
+        { 0x000ff0,   1, 0x04, 0x00000000 },
+        { 0x00179c,   1, 0x04, 0x00000000 },
+        { 0x001228,   1, 0x04, 0x00000400 },
+        { 0x00122c,   1, 0x04, 0x00000300 },
+        { 0x001230,   1, 0x04, 0x00010001 },
+        { 0x0007f8,   1, 0x04, 0x00000000 },
+        { 0x0015b4,   1, 0x04, 0x00000001 },
+        { 0x0015cc,   1, 0x04, 0x00000000 },
+        { 0x001534,   1, 0x04, 0x00000000 },
+        { 0x000fb0,   1, 0x04, 0x00000000 },
+        { 0x0015d0,   1, 0x04, 0x00000000 },
+        { 0x00153c,   1, 0x04, 0x00000000 },
+        { 0x0016b4,   1, 0x04, 0x00000003 },
+        { 0x000fbc,   4, 0x04, 0x0000ffff },
+        { 0x000df8,   2, 0x04, 0x00000000 },
+        { 0x001948,   1, 0x04, 0x00000000 },
+        { 0x001970,   1, 0x04, 0x00000001 },
+        { 0x00161c,   1, 0x04, 0x000009f0 },
+        { 0x000dcc,   1, 0x04, 0x00000010 },
+        { 0x00163c,   1, 0x04, 0x00000000 },
+        { 0x0015e4,   1, 0x04, 0x00000000 },
+        { 0x001160,  32, 0x04, 0x25e00040 },
+        { 0x001880,  32, 0x04, 0x00000000 },
+        { 0x000f84,   2, 0x04, 0x00000000 },
+        { 0x0017c8,   2, 0x04, 0x00000000 },
+        { 0x0017d0,   1, 0x04, 0x000000ff },
+        { 0x0017d4,   1, 0x04, 0xffffffff },
+        { 0x0017d8,   1, 0x04, 0x00000002 },
+        { 0x0017dc,   1, 0x04, 0x00000000 },
+        { 0x0015f4,   2, 0x04, 0x00000000 },
+        { 0x001434,   2, 0x04, 0x00000000 },
+        { 0x000d74,   1, 0x04, 0x00000000 },
+        { 0x000dec,   1, 0x04, 0x00000001 },
+        { 0x0013a4,   1, 0x04, 0x00000000 },
+        { 0x001318,   1, 0x04, 0x00000001 },
+        { 0x001644,   1, 0x04, 0x00000000 },
+        { 0x000748,   1, 0x04, 0x00000000 },
+        { 0x000de8,   1, 0x04, 0x00000000 },
+        { 0x001648,   1, 0x04, 0x00000000 },
+        { 0x0012a4,   1, 0x04, 0x00000000 },
+        { 0x001120,   4, 0x04, 0x00000000 },
+        { 0x001118,   1, 0x04, 0x00000000 },
+        { 0x00164c,   1, 0x04, 0x00000000 },
+        { 0x001658,   1, 0x04, 0x00000000 },
+        { 0x001910,   1, 0x04, 0x00000290 },
+        { 0x001518,   1, 0x04, 0x00000000 },
+        { 0x00165c,   1, 0x04, 0x00000001 },
+        { 0x001520,   1, 0x04, 0x00000000 },
+        { 0x001604,   1, 0x04, 0x00000000 },
+        { 0x001570,   1, 0x04, 0x00000000 },
+        { 0x0013b0,   2, 0x04, 0x3f800000 },
+        { 0x00020c,   1, 0x04, 0x00000000 },
+        { 0x001670,   1, 0x04, 0x30201000 },
+        { 0x001674,   1, 0x04, 0x70605040 },
+        { 0x001678,   1, 0x04, 0xb8a89888 },
+        { 0x00167c,   1, 0x04, 0xf8e8d8c8 },
+        { 0x00166c,   1, 0x04, 0x00000000 },
+        { 0x001680,   1, 0x04, 0x00ffff00 },
+        { 0x0012d0,   1, 0x04, 0x00000003 },
+        { 0x0012d4,   1, 0x04, 0x00000002 },
+        { 0x001684,   2, 0x04, 0x00000000 },
+        { 0x000dac,   2, 0x04, 0x00001b02 },
+        { 0x000db4,   1, 0x04, 0x00000000 },
+        { 0x00168c,   1, 0x04, 0x00000000 },
+        { 0x0015bc,   1, 0x04, 0x00000000 },
+        { 0x00156c,   1, 0x04, 0x00000000 },
+        { 0x00187c,   1, 0x04, 0x00000000 },
+        { 0x001110,   1, 0x04, 0x00000001 },
+        { 0x000dc0,   3, 0x04, 0x00000000 },
+        { 0x001234,   1, 0x04, 0x00000000 },
+        { 0x001690,   1, 0x04, 0x00000000 },
+        { 0x0012ac,   1, 0x04, 0x00000001 },
+        { 0x0002c4,   1, 0x04, 0x00000000 },
+        { 0x000790,   5, 0x04, 0x00000000 },
+        { 0x00077c,   1, 0x04, 0x00000000 },
+        { 0x001000,   1, 0x04, 0x00000010 },
+        { 0x0010fc,   1, 0x04, 0x00000000 },
+        { 0x001290,   1, 0x04, 0x00000000 },
+        { 0x000218,   1, 0x04, 0x00000010 },
+        { 0x0012d8,   1, 0x04, 0x00000000 },
+        { 0x0012dc,   1, 0x04, 0x00000010 },
+        { 0x000d94,   1, 0x04, 0x00000001 },
+        { 0x00155c,   2, 0x04, 0x00000000 },
+        { 0x001564,   1, 0x04, 0x00000fff },
+        { 0x001574,   2, 0x04, 0x00000000 },
+        { 0x00157c,   1, 0x04, 0x000fffff },
+        { 0x001354,   1, 0x04, 0x00000000 },
+        { 0x001610,   1, 0x04, 0x00000012 },
+        { 0x001608,   2, 0x04, 0x00000000 },
+        { 0x00260c,   1, 0x04, 0x00000000 },
+        { 0x0007ac,   1, 0x04, 0x00000000 },
+        { 0x00162c,   1, 0x04, 0x00000003 },
+        { 0x000210,   1, 0x04, 0x00000000 },
+        { 0x000320,   1, 0x04, 0x00000000 },
+        { 0x000324,   6, 0x04, 0x3f800000 },
+        { 0x000750,   1, 0x04, 0x00000000 },
+        { 0x000760,   1, 0x04, 0x39291909 },
+        { 0x000764,   1, 0x04, 0x79695949 },
+        { 0x000768,   1, 0x04, 0xb9a99989 },
+        { 0x00076c,   1, 0x04, 0xf9e9d9c9 },
+        { 0x000770,   1, 0x04, 0x30201000 },
+        { 0x000774,   1, 0x04, 0x70605040 },
+        { 0x000778,   1, 0x04, 0x00009080 },
+        { 0x000780,   1, 0x04, 0x39291909 },
+        { 0x000784,   1, 0x04, 0x79695949 },
+        { 0x000788,   1, 0x04, 0xb9a99989 },
+        { 0x00078c,   1, 0x04, 0xf9e9d9c9 },
+        { 0x0007d0,   1, 0x04, 0x30201000 },
+        { 0x0007d4,   1, 0x04, 0x70605040 },
+        { 0x0007d8,   1, 0x04, 0x00009080 },
+        { 0x00037c,   1, 0x04, 0x00000001 },
+        { 0x000740,   2, 0x04, 0x00000000 },
+        { 0x002600,   1, 0x04, 0x00000000 },
+        { 0x001918,   1, 0x04, 0x00000000 },
+        { 0x00191c,   1, 0x04, 0x00000900 },
+        { 0x001920,   1, 0x04, 0x00000405 },
+        { 0x001308,   1, 0x04, 0x00000001 },
+        { 0x001924,   1, 0x04, 0x00000000 },
+        { 0x0013ac,   1, 0x04, 0x00000000 },
+        { 0x00192c,   1, 0x04, 0x00000001 },
+        { 0x00193c,   1, 0x04, 0x00002c1c },
+        { 0x000d7c,   1, 0x04, 0x00000000 },
+        { 0x000f8c,   1, 0x04, 0x00000000 },
+        { 0x0002c0,   1, 0x04, 0x00000001 },
+        { 0x001510,   1, 0x04, 0x00000000 },
+        { 0x001940,   1, 0x04, 0x00000000 },
+        { 0x000ff4,   2, 0x04, 0x00000000 },
+        { 0x00194c,   2, 0x04, 0x00000000 },
+        { 0x001968,   1, 0x04, 0x00000000 },
+        { 0x001590,   1, 0x04, 0x0000003f },
+        { 0x0007e8,   4, 0x04, 0x00000000 },
+        { 0x00196c,   1, 0x04, 0x00000011 },
+        { 0x0002e4,   1, 0x04, 0x0000b001 },
+        { 0x00036c,   2, 0x04, 0x00000000 },
+        { 0x00197c,   1, 0x04, 0x00000000 },
+        { 0x000fcc,   2, 0x04, 0x00000000 },
+        { 0x0002d8,   1, 0x04, 0x00000040 },
+        { 0x001980,   1, 0x04, 0x00000080 },
+        { 0x001504,   1, 0x04, 0x00000080 },
+        { 0x001984,   1, 0x04, 0x00000000 },
+        { 0x000300,   1, 0x04, 0x00000001 },
+        { 0x0013a8,   1, 0x04, 0x00000000 },
+        { 0x0012ec,   1, 0x04, 0x00000000 },
+        { 0x001310,   1, 0x04, 0x00000000 },
+        { 0x001314,   1, 0x04, 0x00000001 },
+        { 0x001380,   1, 0x04, 0x00000000 },
+        { 0x001384,   4, 0x04, 0x00000001 },
+        { 0x001394,   1, 0x04, 0x00000000 },
+        { 0x00139c,   1, 0x04, 0x00000000 },
+        { 0x001398,   1, 0x04, 0x00000000 },
+        { 0x001594,   1, 0x04, 0x00000000 },
+        { 0x001598,   4, 0x04, 0x00000001 },
+        { 0x000f54,   3, 0x04, 0x00000000 },
+        { 0x0019bc,   1, 0x04, 0x00000000 },
+        { 0x000f9c,   2, 0x04, 0x00000000 },
+        { 0x0012cc,   1, 0x04, 0x00000000 },
+        { 0x0012e8,   1, 0x04, 0x00000000 },
+        { 0x00130c,   1, 0x04, 0x00000001 },
+        { 0x001360,   8, 0x04, 0x00000000 },
+        { 0x00133c,   2, 0x04, 0x00000001 },
+        { 0x001344,   1, 0x04, 0x00000002 },
+        { 0x001348,   2, 0x04, 0x00000001 },
+        { 0x001350,   1, 0x04, 0x00000002 },
+        { 0x001358,   1, 0x04, 0x00000001 },
+        { 0x0012e4,   1, 0x04, 0x00000000 },
+        { 0x00131c,   4, 0x04, 0x00000000 },
+        { 0x0019c0,   1, 0x04, 0x00000000 },
+        { 0x001140,   1, 0x04, 0x00000000 },
+        { 0x0019c4,   1, 0x04, 0x00000000 },
+        { 0x0019c8,   1, 0x04, 0x00001500 },
+        { 0x00135c,   1, 0x04, 0x00000000 },
+        { 0x000f90,   1, 0x04, 0x00000000 },
+        { 0x0019e0,   8, 0x04, 0x00000001 },
+        { 0x0019cc,   1, 0x04, 0x00000001 },
+        { 0x0015b8,   1, 0x04, 0x00000000 },
+        { 0x001a00,   1, 0x04, 0x00001111 },
+        { 0x001a04,   7, 0x04, 0x00000000 },
+        { 0x000d6c,   2, 0x04, 0xffff0000 },
+        { 0x0010f8,   1, 0x04, 0x00001010 },
+        { 0x000d80,   5, 0x04, 0x00000000 },
+        { 0x000da0,   1, 0x04, 0x00000000 },
+        { 0x0007a4,   2, 0x04, 0x00000000 },
+        { 0x001508,   1, 0x04, 0x80000000 },
+        { 0x00150c,   1, 0x04, 0x40000000 },
+        { 0x001668,   1, 0x04, 0x00000000 },
+        { 0x000318,   2, 0x04, 0x00000008 },
+        { 0x000d9c,   1, 0x04, 0x00000001 },
+        { 0x000ddc,   1, 0x04, 0x00000002 },
+        { 0x000374,   1, 0x04, 0x00000000 },
+        { 0x000378,   1, 0x04, 0x00000020 },
+        { 0x0007dc,   1, 0x04, 0x00000000 },
+        { 0x00074c,   1, 0x04, 0x00000055 },
+        { 0x001420,   1, 0x04, 0x00000003 },
+        { 0x0017bc,   2, 0x04, 0x00000000 },
+        { 0x0017c4,   1, 0x04, 0x00000001 },
+        { 0x001008,   1, 0x04, 0x00000008 },
+        { 0x00100c,   1, 0x04, 0x00000040 },
+        { 0x001010,   1, 0x04, 0x0000012c },
+        { 0x000d60,   1, 0x04, 0x00000040 },
+        { 0x00075c,   1, 0x04, 0x00000003 },
+        { 0x001018,   1, 0x04, 0x00000020 },
+        { 0x00101c,   1, 0x04, 0x00000001 },
+        { 0x001020,   1, 0x04, 0x00000020 },
+        { 0x001024,   1, 0x04, 0x00000001 },
+        { 0x001444,   3, 0x04, 0x00000000 },
+        { 0x000360,   1, 0x04, 0x20164010 },
+        { 0x000364,   1, 0x04, 0x00000020 },
+        { 0x000368,   1, 0x04, 0x00000000 },
+        { 0x000de4,   1, 0x04, 0x00000000 },
+        { 0x000204,   1, 0x04, 0x00000006 },
+        { 0x000208,   1, 0x04, 0x00000000 },
+        { 0x0002cc,   2, 0x04, 0x003fffff },
+        { 0x001220,   1, 0x04, 0x00000005 },
+        { 0x000fdc,   1, 0x04, 0x00000000 },
+        { 0x000f98,   1, 0x04, 0x00400008 },
+        { 0x001284,   1, 0x04, 0x08000080 },
+        { 0x001450,   1, 0x04, 0x00400008 },
+        { 0x001454,   1, 0x04, 0x08000080 },
+        { 0x000214,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_pack
+nvf0_grctx_pack_mthd[] = {
+        { nvf0_grctx_init_a197_0, 0xa197 },
+        { nvc0_grctx_init_902d_0, 0x902d },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvf0_grctx_init_fe_0[] = {
         { 0x404004,   8, 0x04, 0x00000000 },
         { 0x404024,   1, 0x04, 0x0000e000 },
         { 0x404028,   8, 0x04, 0x00000000 },
@@ -50,8 +620,8 @@ nvf0_grctx_init_unk40xx[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvf0_grctx_init_unk44xx[] = {
+const struct nvc0_graph_init
+nvf0_grctx_init_pri_0[] = {
         { 0x404404,  12, 0x04, 0x00000000 },
         { 0x404438,   1, 0x04, 0x00000000 },
         { 0x404460,   2, 0x04, 0x00000000 },
@@ -62,23 +632,18 @@ nvf0_grctx_init_unk44xx[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvf0_grctx_init_unk5bxx[] = {
+const struct nvc0_graph_init
+nvf0_grctx_init_cwd_0[] = {
         { 0x405b00,   1, 0x04, 0x00000000 },
         { 0x405b10,   1, 0x04, 0x00001000 },
         { 0x405b20,   1, 0x04, 0x04000000 },
         {}
 };
 
-struct nvc0_graph_init
-nvf0_grctx_init_unk60xx[] = {
+static const struct nvc0_graph_init
+nvf0_grctx_init_pd_0[] = {
         { 0x406020,   1, 0x04, 0x034103c1 },
         { 0x406028,   4, 0x04, 0x00000001 },
-        {}
-};
-
-static struct nvc0_graph_init
-nvf0_grctx_init_unk64xx[] = {
         { 0x4064a8,   1, 0x04, 0x00000000 },
         { 0x4064ac,   1, 0x04, 0x00003fff },
         { 0x4064b0,   3, 0x04, 0x00000000 },
@@ -90,8 +655,8 @@ nvf0_grctx_init_unk64xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nvf0_grctx_init_unk88xx[] = {
+static const struct nvc0_graph_init
+nvf0_grctx_init_be_0[] = {
         { 0x408800,   1, 0x04, 0x12802a3c },
         { 0x408804,   1, 0x04, 0x00000040 },
         { 0x408808,   1, 0x04, 0x1003e005 },
@@ -103,22 +668,23 @@ nvf0_grctx_init_unk88xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nvf0_grctx_init_gpc_0[] = {
-        { 0x418380,   1, 0x04, 0x00000016 },
-        { 0x418400,   1, 0x04, 0x38004e00 },
-        { 0x418404,   1, 0x04, 0x71e0ffff },
-        { 0x41840c,   1, 0x04, 0x00001008 },
-        { 0x418410,   1, 0x04, 0x0fff0fff },
-        { 0x418414,   1, 0x04, 0x02200fff },
-        { 0x418450,   6, 0x04, 0x00000000 },
-        { 0x418468,   1, 0x04, 0x00000001 },
-        { 0x41846c,   2, 0x04, 0x00000000 },
-        { 0x418600,   1, 0x04, 0x0000001f },
-        { 0x418684,   1, 0x04, 0x0000000f },
-        { 0x418700,   1, 0x04, 0x00000002 },
-        { 0x418704,   1, 0x04, 0x00000080 },
-        { 0x418708,   3, 0x04, 0x00000000 },
+static const struct nvc0_graph_pack
+nvf0_grctx_pack_hub[] = {
+        { nvc0_grctx_init_main_0 },
+        { nvf0_grctx_init_fe_0 },
+        { nvf0_grctx_init_pri_0 },
+        { nve4_grctx_init_memfmt_0 },
+        { nve4_grctx_init_ds_0 },
+        { nvf0_grctx_init_cwd_0 },
+        { nvf0_grctx_init_pd_0 },
+        { nvc0_grctx_init_rstr2d_0 },
+        { nve4_grctx_init_scc_0 },
+        { nvf0_grctx_init_be_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvf0_grctx_init_setup_0[] = {
         { 0x418800,   1, 0x04, 0x7006860a },
         { 0x418808,   1, 0x04, 0x00000000 },
         { 0x41880c,   1, 0x04, 0x00000030 },
@@ -129,36 +695,31 @@ nvf0_grctx_init_gpc_0[] = {
         { 0x4188e0,   1, 0x04, 0x01000000 },
         { 0x4188e8,   5, 0x04, 0x00000000 },
         { 0x4188fc,   1, 0x04, 0x20100018 },
-        { 0x41891c,   1, 0x04, 0x00ff00ff },
-        { 0x418924,   1, 0x04, 0x00000000 },
-        { 0x418928,   1, 0x04, 0x00ffff00 },
-        { 0x41892c,   1, 0x04, 0x0000ff00 },
-        { 0x418b00,   1, 0x04, 0x00000006 },
-        { 0x418b08,   1, 0x04, 0x0a418820 },
-        { 0x418b0c,   1, 0x04, 0x062080e6 },
-        { 0x418b10,   1, 0x04, 0x020398a4 },
-        { 0x418b14,   1, 0x04, 0x0e629062 },
-        { 0x418b18,   1, 0x04, 0x0a418820 },
-        { 0x418b1c,   1, 0x04, 0x000000e6 },
-        { 0x418bb8,   1, 0x04, 0x00000103 },
-        { 0x418c08,   1, 0x04, 0x00000001 },
-        { 0x418c10,   8, 0x04, 0x00000000 },
-        { 0x418c40,   1, 0x04, 0xffffffff },
-        { 0x418c6c,   1, 0x04, 0x00000001 },
-        { 0x418c80,   1, 0x04, 0x20200004 },
-        { 0x418c8c,   1, 0x04, 0x00000001 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvf0_grctx_init_gpc_unk_2[] = {
         { 0x418d24,   1, 0x04, 0x00000000 },
-        { 0x419000,   1, 0x04, 0x00000780 },
-        { 0x419004,   2, 0x04, 0x00000000 },
-        { 0x419014,   1, 0x04, 0x00000004 },
         {}
 };
 
-static struct nvc0_graph_init
-nvf0_grctx_init_tpc[] = {
-        { 0x419848,   1, 0x04, 0x00000000 },
-        { 0x419864,   1, 0x04, 0x00000129 },
-        { 0x419888,   1, 0x04, 0x00000000 },
+static const struct nvc0_graph_pack
+nvf0_grctx_pack_gpc[] = {
+        { nvc0_grctx_init_gpc_unk_0 },
+        { nvd9_grctx_init_prop_0 },
+        { nvd9_grctx_init_gpc_unk_1 },
+        { nvf0_grctx_init_setup_0 },
+        { nvc0_grctx_init_zcull_0 },
+        { nvd9_grctx_init_crstr_0 },
+        { nve4_grctx_init_gpm_0 },
+        { nvf0_grctx_init_gpc_unk_2 },
+        { nvc0_grctx_init_gcc_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvf0_grctx_init_tex_0[] = {
         { 0x419a00,   1, 0x04, 0x000000f0 },
         { 0x419a04,   1, 0x04, 0x00000001 },
         { 0x419a08,   1, 0x04, 0x00000021 },
@@ -169,14 +730,29 @@ nvf0_grctx_init_tpc[] = {
         { 0x419a20,   1, 0x04, 0x00020800 },
         { 0x419a30,   1, 0x04, 0x00000001 },
         { 0x419ac4,   1, 0x04, 0x0037f440 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvf0_grctx_init_mpc_0[] = {
         { 0x419c00,   1, 0x04, 0x0000001a },
         { 0x419c04,   1, 0x04, 0x80000006 },
         { 0x419c08,   1, 0x04, 0x00000002 },
         { 0x419c20,   1, 0x04, 0x00000000 },
         { 0x419c24,   1, 0x04, 0x00084210 },
         { 0x419c28,   1, 0x04, 0x3efbefbe },
+        {}
+};
+
+const struct nvc0_graph_init
+nvf0_grctx_init_l1c_0[] = {
         { 0x419ce8,   1, 0x04, 0x00000000 },
         { 0x419cf4,   1, 0x04, 0x00000203 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvf0_grctx_init_sm_0[] = {
         { 0x419e04,   1, 0x04, 0x00000000 },
         { 0x419e08,   1, 0x04, 0x0000001d },
         { 0x419e0c,   1, 0x04, 0x00000000 },
@@ -189,8 +765,8 @@ nvf0_grctx_init_tpc[] = {
         { 0x419e5c,   3, 0x04, 0x00000000 },
         { 0x419e68,   1, 0x04, 0x00000002 },
         { 0x419e6c,  12, 0x04, 0x00000000 },
-        { 0x419eac,   1, 0x04, 0x00001fcf },
-        { 0x419eb0,   1, 0x04, 0x0db00da0 },
+        { 0x419eac,   1, 0x04, 0x00001f8f },
+        { 0x419eb0,   1, 0x04, 0x0db00d2f },
         { 0x419eb8,   1, 0x04, 0x00000000 },
         { 0x419ec8,   1, 0x04, 0x0001304f },
         { 0x419f30,   4, 0x04, 0x00000000 },
@@ -203,24 +779,36 @@ nvf0_grctx_init_tpc[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nvf0_grctx_init_unk[] = {
-        { 0x41be24,   1, 0x04, 0x00000006 },
+static const struct nvc0_graph_pack
+nvf0_grctx_pack_tpc[] = {
+        { nvd7_grctx_init_pe_0 },
+        { nvf0_grctx_init_tex_0 },
+        { nvf0_grctx_init_mpc_0 },
+        { nvf0_grctx_init_l1c_0 },
+        { nvf0_grctx_init_sm_0 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvf0_grctx_init_cbm_0[] = {
         { 0x41bec0,   1, 0x04, 0x10000000 },
         { 0x41bec4,   1, 0x04, 0x00037f7f },
         { 0x41bee4,   1, 0x04, 0x00000000 },
-        { 0x41bf00,   1, 0x04, 0x0a418820 },
-        { 0x41bf04,   1, 0x04, 0x062080e6 },
-        { 0x41bf08,   1, 0x04, 0x020398a4 },
-        { 0x41bf0c,   1, 0x04, 0x0e629062 },
-        { 0x41bf10,   1, 0x04, 0x0a418820 },
-        { 0x41bf14,   1, 0x04, 0x000000e6 },
-        { 0x41bfd0,   1, 0x04, 0x00900103 },
-        { 0x41bfe0,   1, 0x04, 0x00400001 },
-        { 0x41bfe4,   1, 0x04, 0x00000000 },
         {}
 };
 
+static const struct nvc0_graph_pack
+nvf0_grctx_pack_ppc[] = {
+        { nve4_grctx_init_pes_0 },
+        { nvf0_grctx_init_cbm_0 },
+        { nvd7_grctx_init_wwdx_0 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH context implementation
+ ******************************************************************************/
+
 static void
 nvf0_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
 {
@@ -273,39 +861,6 @@ nvf0_grctx_generate_mods(struct nvc0_graph_priv *priv, struct nvc0_grctx *info)
         mmio_list(0x17e920, 0x00090a05, 0, 0);
 }
 
-static struct nvc0_graph_init *
-nvf0_grctx_init_hub[] = {
-        nvc0_grctx_init_base,
-        nvf0_grctx_init_unk40xx,
-        nvf0_grctx_init_unk44xx,
-        nve4_grctx_init_unk46xx,
-        nve4_grctx_init_unk47xx,
-        nve4_grctx_init_unk58xx,
-        nvf0_grctx_init_unk5bxx,
-        nvf0_grctx_init_unk60xx,
-        nvf0_grctx_init_unk64xx,
-        nve4_grctx_init_unk80xx,
-        nvf0_grctx_init_unk88xx,
-        NULL
-};
-
-struct nvc0_graph_init *
-nvf0_grctx_init_gpc[] = {
-        nvf0_grctx_init_gpc_0,
-        nvc0_grctx_init_gpc_1,
-        nvf0_grctx_init_tpc,
-        nvf0_grctx_init_unk,
-        NULL
-};
-
-static struct nvc0_graph_mthd
-nvf0_grctx_init_mthd[] = {
-        { 0xa197, nvc1_grctx_init_9097, },
-        { 0x902d, nvc0_grctx_init_902d, },
-        { 0x902d, nvc0_grctx_init_mthd_magic, },
-        {}
-};
-
 struct nouveau_oclass *
 nvf0_grctx_oclass = &(struct nvc0_grctx_oclass) {
         .base.handle = NV_ENGCTX(GR, 0xf0),
@@ -317,11 +872,14 @@ nvf0_grctx_oclass = &(struct nvc0_grctx_oclass) {
                 .rd32 = _nouveau_graph_context_rd32,
                 .wr32 = _nouveau_graph_context_wr32,
         },
-        .main = nve4_grctx_generate_main,
-        .mods = nvf0_grctx_generate_mods,
-        .unkn = nve4_grctx_generate_unkn,
-        .hub  = nvf0_grctx_init_hub,
-        .gpc  = nvf0_grctx_init_gpc,
-        .icmd = nvc0_grctx_init_icmd,
-        .mthd = nvf0_grctx_init_mthd,
+        .main  = nve4_grctx_generate_main,
+        .mods  = nvf0_grctx_generate_mods,
+        .unkn  = nve4_grctx_generate_unkn,
+        .hub   = nvf0_grctx_pack_hub,
+        .gpc   = nvf0_grctx_pack_gpc,
+        .zcull = nvc0_grctx_pack_zcull,
+        .tpc   = nvf0_grctx_pack_tpc,
+        .ppc   = nvf0_grctx_pack_ppc,
+        .icmd  = nvf0_grctx_pack_icmd,
+        .mthd  = nvf0_grctx_pack_mthd,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/com.fuc b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/com.fuc
index e148961b8075..e37d8106ae1a 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/com.fuc
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/com.fuc
@@ -228,7 +228,7 @@ mmctx_xfer:
                         and $r11 0x1f
                         cmpu b32 $r11 0x10
                         bra ne #mmctx_fini_wait
-                mov $r10 2                              // DONE_MMCTX
+                mov $r10 5                      // DONE_MMCTX
                 call(wait_donez)
                 bra #mmctx_done
         mmctx_stop:
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpc.fuc b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpc.fuc
index 96cbcea3b2c9..2f7345f7fe07 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpc.fuc
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpc.fuc
@@ -78,7 +78,12 @@ error:
 //
 init:
         clear b32 $r0
-        mov $sp $r0
+
+        // setup stack
+        nv_iord($r1, NV_PGRAPH_GPCX_GPCCS_CAPS, 0)
+        extr $r1 $r1 9:17
+        shl b32 $r1 8
+        mov $sp $r1
 
         // enable fifo access
         mov $r2 NV_PGRAPH_GPCX_GPCCS_ACCESS_FIFO
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcgm107.fuc5 b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcgm107.fuc5
new file mode 100644
index 000000000000..e730603891d7
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcgm107.fuc5
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#define NV_PGRAPH_GPCX_UNK__SIZE                                     0x00000002
+
+#define CHIPSET GK208
+#include "macros.fuc"
+
+.section #gm107_grgpc_data
+#define INCLUDE_DATA
+#include "com.fuc"
+#include "gpc.fuc"
+#undef INCLUDE_DATA
+
+.section #gm107_grgpc_code
+#define INCLUDE_CODE
+bra #init
+#include "com.fuc"
+#include "gpc.fuc"
+.align 256
+#undef INCLUDE_CODE
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcgm107.fuc5.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcgm107.fuc5.h
new file mode 100644
index 000000000000..6d53b67dd3c4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcgm107.fuc5.h
@@ -0,0 +1,473 @@
+uint32_t gm107_grgpc_data[] = {
+/* 0x0000: gpc_mmio_list_head */
+        0x0000006c,
+/* 0x0004: gpc_mmio_list_tail */
+/* 0x0004: tpc_mmio_list_head */
+        0x0000006c,
+/* 0x0008: tpc_mmio_list_tail */
+/* 0x0008: unk_mmio_list_head */
+        0x0000006c,
+/* 0x000c: unk_mmio_list_tail */
+        0x0000006c,
+/* 0x0010: gpc_id */
+        0x00000000,
+/* 0x0014: tpc_count */
+        0x00000000,
+/* 0x0018: tpc_mask */
+        0x00000000,
+/* 0x001c: unk_count */
+        0x00000000,
+/* 0x0020: unk_mask */
+        0x00000000,
+/* 0x0024: cmd_queue */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
+
+uint32_t gm107_grgpc_code[] = {
+        0x03140ef5,
+/* 0x0004: queue_put */
+        0x9800d898,
+        0x86f001d9,
+        0xf489a408,
+        0x020f0b1b,
+        0x0002f87e,
+/* 0x001a: queue_put_next */
+        0x98c400f8,
+        0x0384b607,
+        0xb6008dbb,
+        0x8eb50880,
+        0x018fb500,
+        0xf00190b6,
+        0xd9b50f94,
+/* 0x0037: queue_get */
+        0xf400f801,
+        0xd8980131,
+        0x01d99800,
+        0x0bf489a4,
+        0x0789c421,
+        0xbb0394b6,
+        0x90b6009d,
+        0x009e9808,
+        0xb6019f98,
+        0x84f00180,
+        0x00d8b50f,
+/* 0x0063: queue_get_done */
+        0xf80132f4,
+/* 0x0065: nv_rd32 */
+        0xf0ecb200,
+        0x00801fc9,
+        0x0cf601ca,
+/* 0x0073: nv_rd32_wait */
+        0x8c04bd00,
+        0xcf01ca00,
+        0xccc800cc,
+        0xf61bf41f,
+        0xec7e060a,
+        0x008f0000,
+        0xffcf01cb,
+/* 0x008f: nv_wr32 */
+        0x8000f800,
+        0xf601cc00,
+        0x04bd000f,
+        0xc9f0ecb2,
+        0x1ec9f01f,
+        0x01ca0080,
+        0xbd000cf6,
+/* 0x00a9: nv_wr32_wait */
+        0xca008c04,
+        0x00cccf01,
+        0xf41fccc8,
+        0x00f8f61b,
+/* 0x00b8: wait_donez */
+        0x99f094bd,
+        0x37008000,
+        0x0009f602,
+        0x008004bd,
+        0x0af60206,
+/* 0x00cf: wait_donez_ne */
+        0x8804bd00,
+        0xcf010000,
+        0x8aff0088,
+        0xf61bf488,
+        0x99f094bd,
+        0x17008000,
+        0x0009f602,
+        0x00f804bd,
+/* 0x00ec: wait_doneo */
+        0x99f094bd,
+        0x37008000,
+        0x0009f602,
+        0x008004bd,
+        0x0af60206,
+/* 0x0103: wait_doneo_e */
+        0x8804bd00,
+        0xcf010000,
+        0x8aff0088,
+        0xf60bf488,
+        0x99f094bd,
+        0x17008000,
+        0x0009f602,
+        0x00f804bd,
+/* 0x0120: mmctx_size */
+/* 0x0122: nv_mmctx_size_loop */
+        0xe89894bd,
+        0x1a85b600,
+        0xb60180b6,
+        0x98bb0284,
+        0x04e0b600,
+        0x1bf4efa4,
+        0xf89fb2ec,
+/* 0x013d: mmctx_xfer */
+        0xf094bd00,
+        0x00800199,
+        0x09f60237,
+        0xbd04bd00,
+        0x05bbfd94,
+        0x800f0bf4,
+        0xf601c400,
+        0x04bd000b,
+/* 0x015f: mmctx_base_disabled */
+        0xfd0099f0,
+        0x0bf405ee,
+        0xc6008018,
+        0x000ef601,
+        0x008004bd,
+        0x0ff601c7,
+        0xf004bd00,
+/* 0x017a: mmctx_multi_disabled */
+        0xabc80199,
+        0x10b4b600,
+        0xc80cb9f0,
+        0xe4b601ae,
+        0x05befd11,
+        0x01c50080,
+        0xbd000bf6,
+/* 0x0195: mmctx_exec_loop */
+/* 0x0195: mmctx_wait_free */
+        0xc5008e04,
+        0x00eecf01,
+        0xf41fe4f0,
+        0xce98f60b,
+        0x05e9fd00,
+        0x01c80080,
+        0xbd000ef6,
+        0x04c0b604,
+        0x1bf4cda4,
+        0x02abc8df,
+/* 0x01bf: mmctx_fini_wait */
+        0x8b1c1bf4,
+        0xcf01c500,
+        0xb4f000bb,
+        0x10b4b01f,
+        0x0af31bf4,
+        0x00b87e05,
+        0x250ef400,
+/* 0x01d8: mmctx_stop */
+        0xb600abc8,
+        0xb9f010b4,
+        0x12b9f00c,
+        0x01c50080,
+        0xbd000bf6,
+/* 0x01ed: mmctx_stop_wait */
+        0xc5008b04,
+        0x00bbcf01,
+        0xf412bbc8,
+/* 0x01fa: mmctx_done */
+        0x94bdf61b,
+        0x800199f0,
+        0xf6021700,
+        0x04bd0009,
+/* 0x020a: strand_wait */
+        0xa0f900f8,
+        0xb87e020a,
+        0xa0fc0000,
+/* 0x0216: strand_pre */
+        0x0c0900f8,
+        0x024afc80,
+        0xbd0009f6,
+        0x020a7e04,
+/* 0x0227: strand_post */
+        0x0900f800,
+        0x4afc800d,
+        0x0009f602,
+        0x0a7e04bd,
+        0x00f80002,
+/* 0x0238: strand_set */
+        0xfc800f0c,
+        0x0cf6024f,
+        0x0c04bd00,
+        0x4afc800b,
+        0x000cf602,
+        0xfc8004bd,
+        0x0ef6024f,
+        0x0c04bd00,
+        0x4afc800a,
+        0x000cf602,
+        0x0a7e04bd,
+        0x00f80002,
+/* 0x0268: strand_ctx_init */
+        0x99f094bd,
+        0x37008003,
+        0x0009f602,
+        0x167e04bd,
+        0x030e0002,
+        0x0002387e,
+        0xfc80c4bd,
+        0x0cf60247,
+        0x0c04bd00,
+        0x4afc8001,
+        0x000cf602,
+        0x0a7e04bd,
+        0x0c920002,
+        0x46fc8001,
+        0x000cf602,
+        0x020c04bd,
+        0x024afc80,
+        0xbd000cf6,
+        0x020a7e04,
+        0x02277e00,
+        0x42008800,
+        0x20008902,
+        0x0099cf02,
+/* 0x02c7: ctx_init_strand_loop */
+        0xf608fe95,
+        0x8ef6008e,
+        0x808acf40,
+        0xb606a5b6,
+        0xeabb01a0,
+        0x0480b600,
+        0xf40192b6,
+        0xe4b6e81b,
+        0xf2efbc08,
+        0x99f094bd,
+        0x17008003,
+        0x0009f602,
+        0x00f804bd,
+/* 0x02f8: error */
+        0xffb2e0f9,
+        0x4098148e,
+        0x00008f7e,
+        0xffb2010f,
+        0x409c1c8e,
+        0x00008f7e,
+        0x00f8e0fc,
+/* 0x0314: init */
+        0x004104bd,
+        0x0011cf42,
+        0x010911e7,
+        0xfe0814b6,
+        0x02020014,
+        0xf6120040,
+        0x04bd0002,
+        0xfe047241,
+        0x00400010,
+        0x0000f607,
+        0x040204bd,
+        0xf6040040,
+        0x04bd0002,
+        0x821031f4,
+        0xcf018200,
+        0x01030022,
+        0xbb1f24f0,
+        0x32b60432,
+        0x0502b501,
+        0x820603b5,
+        0xcf018600,
+        0x02b50022,
+        0x0c308e04,
+        0xbd24bd50,
+/* 0x0377: init_unk_loop */
+        0x7e44bd34,
+        0xb0000065,
+        0x0bf400f6,
+        0xbb010f0e,
+        0x4ffd04f2,
+        0x0130b605,
+/* 0x038c: init_unk_next */
+        0xb60120b6,
+        0x26b004e0,
+        0xe21bf402,
+/* 0x0398: init_unk_done */
+        0xb50703b5,
+        0x00820804,
+        0x22cf0201,
+        0x9534bd00,
+        0x00800825,
+        0x05f601c0,
+        0x8004bd00,
+        0xf601c100,
+        0x04bd0005,
+        0x98000e98,
+        0x207e010f,
+        0x2fbb0001,
+        0x003fbb00,
+        0x98010e98,
+        0x207e020f,
+        0x0e980001,
+        0x00effd05,
+        0xbb002ebb,
+        0x0e98003e,
+        0x030f9802,
+        0x0001207e,
+        0xfd070e98,
+        0x2ebb00ef,
+        0x003ebb00,
+        0x800235b6,
+        0xf601d300,
+        0x04bd0003,
+        0xb60825b6,
+        0x20b60635,
+        0x0130b601,
+        0xb60824b6,
+        0x2fb20834,
+        0x0002687e,
+        0x80003fbb,
+        0xf6020100,
+        0x04bd0003,
+        0x29f024bd,
+        0x3000801f,
+        0x0002f602,
+/* 0x0436: main */
+        0x31f404bd,
+        0x0028f400,
+        0x377e240d,
+        0x01f40000,
+        0x04e4b0f4,
+        0xfe1d18f4,
+        0x06020181,
+        0x12fd20bd,
+        0x01e4b604,
+        0xfe051efd,
+        0x097e0018,
+        0x0ef40005,
+/* 0x0465: main_not_ctx_xfer */
+        0x10ef94d4,
+        0x7e01f5f0,
+        0xf40002f8,
+/* 0x0472: ih */
+        0x80f9c70e,
+        0xf90188fe,
+        0xf990f980,
+        0xf9b0f9a0,
+        0xf9e0f9d0,
+        0x4a04bdf0,
+        0xaacf0200,
+        0x04abc400,
+        0x0d1f0bf4,
+        0x1a004e24,
+        0x4f00eecf,
+        0xffcf1900,
+        0x00047e00,
+        0x40010e00,
+        0x0ef61d00,
+/* 0x04af: ih_no_fifo */
+        0x4004bd00,
+        0x0af60100,
+        0xfc04bd00,
+        0xfce0fcf0,
+        0xfcb0fcd0,
+        0xfc90fca0,
+        0x0088fe80,
+        0x32f480fc,
+/* 0x04cf: hub_barrier_done */
+        0x0f01f800,
+        0x040e9801,
+        0xb204febb,
+        0x94188eff,
+        0x008f7e40,
+/* 0x04e3: ctx_redswitch */
+        0x0f00f800,
+        0x85008020,
+        0x000ff601,
+        0x080e04bd,
+/* 0x04f0: ctx_redswitch_delay */
+        0xf401e2b6,
+        0xf5f1fd1b,
+        0xf5f10800,
+        0x00800200,
+        0x0ff60185,
+        0xf804bd00,
+/* 0x0509: ctx_xfer */
+        0x81008000,
+        0x000ff602,
+        0x11f404bd,
+        0x04e37e07,
+/* 0x0519: ctx_xfer_not_load */
+        0x02167e00,
+        0x8024bd00,
+        0xf60247fc,
+        0x04bd0002,
+        0xb6012cf0,
+        0xfc800320,
+        0x02f6024a,
+        0xf004bd00,
+        0xa5f001ac,
+        0x00008b02,
+        0x040c9850,
+        0xbb0fc4b6,
+        0x0c9800bc,
+        0x010d9800,
+        0x3d7e000e,
+        0xacf00001,
+        0x40008b01,
+        0x040c9850,
+        0xbb0fc4b6,
+        0x0c9800bc,
+        0x020d9801,
+        0x4e060f98,
+        0x3d7e0800,
+        0xacf00001,
+        0x04a5f001,
+        0x5030008b,
+        0xb6040c98,
+        0xbcbb0fc4,
+        0x020c9800,
+        0x98030d98,
+        0x004e080f,
+        0x013d7e02,
+        0x020a7e00,
+        0x0601f400,
+/* 0x05a3: ctx_xfer_post */
+        0x7e0712f4,
+/* 0x05a7: ctx_xfer_done */
+        0x7e000227,
+        0xf80004cf,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnv108.fuc5.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnv108.fuc5.h
index 27dc1280dc10..31922707794f 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnv108.fuc5.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnv108.fuc5.h
@@ -177,7 +177,7 @@ uint32_t nv108_grgpc_code[] = {
         0xb4f000bb,
         0x10b4b01f,
         0x0af31bf4,
-        0x00b87e02,
+        0x00b87e05,
         0x250ef400,
 /* 0x01d8: mmctx_stop */
         0xb600abc8,
@@ -269,186 +269,186 @@ uint32_t nv108_grgpc_code[] = {
         0x00008f7e,
         0x00f8e0fc,
 /* 0x0314: init */
-        0x04fe04bd,
-        0x40020200,
-        0x02f61200,
-        0x4104bd00,
-        0x10fe0465,
-        0x07004000,
-        0xbd0000f6,
-        0x40040204,
-        0x02f60400,
-        0xf404bd00,
-        0x00821031,
-        0x22cf0182,
-        0xf0010300,
-        0x32bb1f24,
-        0x0132b604,
-        0xb50502b5,
-        0x00820603,
-        0x22cf0186,
-        0x0402b500,
-        0x500c308e,
-        0x34bd24bd,
-/* 0x036a: init_unk_loop */
-        0x657e44bd,
-        0xf6b00000,
-        0x0e0bf400,
-        0xf2bb010f,
-        0x054ffd04,
-/* 0x037f: init_unk_next */
-        0xb60130b6,
-        0xe0b60120,
-        0x0126b004,
-/* 0x038b: init_unk_done */
-        0xb5e21bf4,
-        0x04b50703,
-        0x01008208,
-        0x0022cf02,
-        0x259534bd,
-        0xc0008008,
-        0x0005f601,
-        0x008004bd,
-        0x05f601c1,
-        0x9804bd00,
-        0x0f98000e,
-        0x01207e01,
-        0x002fbb00,
-        0x98003fbb,
-        0x0f98010e,
-        0x01207e02,
-        0x050e9800,
-        0xbb00effd,
-        0x3ebb002e,
-        0x020e9800,
-        0x7e030f98,
-        0x98000120,
-        0xeffd070e,
-        0x002ebb00,
-        0xb6003ebb,
-        0x00800235,
-        0x03f601d3,
-        0xb604bd00,
-        0x35b60825,
-        0x0120b606,
-        0xb60130b6,
-        0x34b60824,
-        0x7e2fb208,
-        0xbb000268,
-        0x0080003f,
-        0x03f60201,
-        0xbd04bd00,
-        0x1f29f024,
-        0x02300080,
-        0xbd0002f6,
-/* 0x0429: main */
-        0x0031f404,
-        0x0d0028f4,
-        0x00377e24,
-        0xf401f400,
-        0xf404e4b0,
-        0x81fe1d18,
-        0xbd060201,
-        0x0412fd20,
-        0xfd01e4b6,
-        0x18fe051e,
-        0x04fc7e00,
-        0xd40ef400,
-/* 0x0458: main_not_ctx_xfer */
-        0xf010ef94,
-        0xf87e01f5,
-        0x0ef40002,
-/* 0x0465: ih */
-        0xfe80f9c7,
-        0x80f90188,
-        0xa0f990f9,
-        0xd0f9b0f9,
-        0xf0f9e0f9,
-        0x004a04bd,
-        0x00aacf02,
-        0xf404abc4,
-        0x240d1f0b,
-        0xcf1a004e,
-        0x004f00ee,
-        0x00ffcf19,
-        0x0000047e,
-        0x0040010e,
-        0x000ef61d,
-/* 0x04a2: ih_no_fifo */
-        0x004004bd,
-        0x000af601,
-        0xf0fc04bd,
-        0xd0fce0fc,
-        0xa0fcb0fc,
-        0x80fc90fc,
-        0xfc0088fe,
-        0x0032f480,
-/* 0x04c2: hub_barrier_done */
-        0x010f01f8,
-        0xbb040e98,
-        0xffb204fe,
-        0x4094188e,
-        0x00008f7e,
-/* 0x04d6: ctx_redswitch */
-        0x200f00f8,
-        0x01850080,
-        0xbd000ff6,
-/* 0x04e3: ctx_redswitch_delay */
-        0xb6080e04,
-        0x1bf401e2,
-        0x00f5f1fd,
-        0x00f5f108,
-        0x85008002,
+        0x004104bd,
+        0x0011cf42,
+        0x010911e7,
+        0xfe0814b6,
+        0x02020014,
+        0xf6120040,
+        0x04bd0002,
+        0xfe047241,
+        0x00400010,
+        0x0000f607,
+        0x040204bd,
+        0xf6040040,
+        0x04bd0002,
+        0x821031f4,
+        0xcf018200,
+        0x01030022,
+        0xbb1f24f0,
+        0x32b60432,
+        0x0502b501,
+        0x820603b5,
+        0xcf018600,
+        0x02b50022,
+        0x0c308e04,
+        0xbd24bd50,
+/* 0x0377: init_unk_loop */
+        0x7e44bd34,
+        0xb0000065,
+        0x0bf400f6,
+        0xbb010f0e,
+        0x4ffd04f2,
+        0x0130b605,
+/* 0x038c: init_unk_next */
+        0xb60120b6,
+        0x26b004e0,
+        0xe21bf401,
+/* 0x0398: init_unk_done */
+        0xb50703b5,
+        0x00820804,
+        0x22cf0201,
+        0x9534bd00,
+        0x00800825,
+        0x05f601c0,
+        0x8004bd00,
+        0xf601c100,
+        0x04bd0005,
+        0x98000e98,
+        0x207e010f,
+        0x2fbb0001,
+        0x003fbb00,
+        0x98010e98,
+        0x207e020f,
+        0x0e980001,
+        0x00effd05,
+        0xbb002ebb,
+        0x0e98003e,
+        0x030f9802,
+        0x0001207e,
+        0xfd070e98,
+        0x2ebb00ef,
+        0x003ebb00,
+        0x800235b6,
+        0xf601d300,
+        0x04bd0003,
+        0xb60825b6,
+        0x20b60635,
+        0x0130b601,
+        0xb60824b6,
+        0x2fb20834,
+        0x0002687e,
+        0x80003fbb,
+        0xf6020100,
+        0x04bd0003,
+        0x29f024bd,
+        0x3000801f,
+        0x0002f602,
+/* 0x0436: main */
+        0x31f404bd,
+        0x0028f400,
+        0x377e240d,
+        0x01f40000,
+        0x04e4b0f4,
+        0xfe1d18f4,
+        0x06020181,
+        0x12fd20bd,
+        0x01e4b604,
+        0xfe051efd,
+        0x097e0018,
+        0x0ef40005,
+/* 0x0465: main_not_ctx_xfer */
+        0x10ef94d4,
+        0x7e01f5f0,
+        0xf40002f8,
+/* 0x0472: ih */
+        0x80f9c70e,
+        0xf90188fe,
+        0xf990f980,
+        0xf9b0f9a0,
+        0xf9e0f9d0,
+        0x4a04bdf0,
+        0xaacf0200,
+        0x04abc400,
+        0x0d1f0bf4,
+        0x1a004e24,
+        0x4f00eecf,
+        0xffcf1900,
+        0x00047e00,
+        0x40010e00,
+        0x0ef61d00,
+/* 0x04af: ih_no_fifo */
+        0x4004bd00,
+        0x0af60100,
+        0xfc04bd00,
+        0xfce0fcf0,
+        0xfcb0fcd0,
+        0xfc90fca0,
+        0x0088fe80,
+        0x32f480fc,
+/* 0x04cf: hub_barrier_done */
+        0x0f01f800,
+        0x040e9801,
+        0xb204febb,
+        0x94188eff,
+        0x008f7e40,
+/* 0x04e3: ctx_redswitch */
+        0x0f00f800,
+        0x85008020,
         0x000ff601,
-        0x00f804bd,
-/* 0x04fc: ctx_xfer */
-        0x02810080,
-        0xbd000ff6,
-        0x0711f404,
-        0x0004d67e,
-/* 0x050c: ctx_xfer_not_load */
-        0x0002167e,
-        0xfc8024bd,
-        0x02f60247,
+        0x080e04bd,
+/* 0x04f0: ctx_redswitch_delay */
+        0xf401e2b6,
+        0xf5f1fd1b,
+        0xf5f10800,
+        0x00800200,
+        0x0ff60185,
+        0xf804bd00,
+/* 0x0509: ctx_xfer */
+        0x81008000,
+        0x000ff602,
+        0x11f404bd,
+        0x04e37e07,
+/* 0x0519: ctx_xfer_not_load */
+        0x02167e00,
+        0x8024bd00,
+        0xf60247fc,
+        0x04bd0002,
+        0xb6012cf0,
+        0xfc800320,
+        0x02f6024a,
         0xf004bd00,
-        0x20b6012c,
-        0x4afc8003,
-        0x0002f602,
-        0xacf004bd,
-        0x02a5f001,
-        0x5000008b,
+        0xa5f001ac,
+        0x00008b02,
+        0x040c9850,
+        0xbb0fc4b6,
+        0x0c9800bc,
+        0x010d9800,
+        0x3d7e000e,
+        0xacf00001,
+        0x40008b01,
+        0x040c9850,
+        0xbb0fc4b6,
+        0x0c9800bc,
+        0x020d9801,
+        0x4e060f98,
+        0x3d7e0800,
+        0xacf00001,
+        0x04a5f001,
+        0x5030008b,
         0xb6040c98,
         0xbcbb0fc4,
-        0x000c9800,
-        0x0e010d98,
-        0x013d7e00,
-        0x01acf000,
-        0x5040008b,
-        0xb6040c98,
-        0xbcbb0fc4,
-        0x010c9800,
-        0x98020d98,
-        0x004e060f,
-        0x013d7e08,
-        0x01acf000,
-        0x8b04a5f0,
-        0x98503000,
-        0xc4b6040c,
-        0x00bcbb0f,
-        0x98020c98,
-        0x0f98030d,
-        0x02004e08,
-        0x00013d7e,
-        0x00020a7e,
-        0xf40601f4,
-/* 0x0596: ctx_xfer_post */
-        0x277e0712,
-/* 0x059a: ctx_xfer_done */
-        0xc27e0002,
-        0x00f80004,
-        0x00000000,
-        0x00000000,
-        0x00000000,
+        0x020c9800,
+        0x98030d98,
+        0x004e080f,
+        0x013d7e02,
+        0x020a7e00,
+        0x0601f400,
+/* 0x05a3: ctx_xfer_post */
+        0x7e0712f4,
+/* 0x05a7: ctx_xfer_done */
+        0x7e000227,
+        0xf80004cf,
         0x00000000,
         0x00000000,
         0x00000000,
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvc0.fuc.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvc0.fuc.h
index 0e7b01efae8d..325cc7b7b2fb 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvc0.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvc0.fuc.h
@@ -192,7 +192,7 @@ uint32_t nvc0_grgpc_code[] = {
         0x1fb4f000,
         0xf410b4b0,
         0xa7f0f01b,
-        0xd021f402,
+        0xd021f405,
 /* 0x0223: mmctx_stop */
         0xc82b0ef4,
         0xb4b600ab,
@@ -300,182 +300,182 @@ uint32_t nvc0_grgpc_code[] = {
         0x21f440e3,
         0xf8e0fc9d,
 /* 0x03a1: init */
-        0xfe04bd00,
-        0x27f00004,
-        0x0007f102,
-        0x0003f012,
-        0xbd0002d0,
-        0xd517f104,
-        0x0010fe04,
-        0x070007f1,
+        0xf104bd00,
+        0xf0420017,
+        0x11cf0013,
+        0x0911e700,
+        0x0814b601,
+        0xf00014fe,
+        0x07f10227,
+        0x03f01200,
+        0x0002d000,
+        0x17f104bd,
+        0x10fe04e6,
+        0x0007f100,
+        0x0003f007,
+        0xbd0000d0,
+        0x0427f004,
+        0x040007f1,
         0xd00003f0,
-        0x04bd0000,
-        0xf10427f0,
-        0xf0040007,
-        0x02d00003,
-        0xf404bd00,
-        0x27f11031,
-        0x23f08200,
-        0x0022cf01,
-        0xf00137f0,
-        0x32bb1f24,
-        0x0132b604,
-        0x80050280,
-        0x27f10603,
-        0x23f08600,
-        0x0022cf01,
-        0xf1040280,
-        0xf0010027,
-        0x22cf0223,
-        0x9534bd00,
-        0x07f10825,
-        0x03f0c000,
-        0x0005d001,
-        0x07f104bd,
-        0x03f0c100,
-        0x0005d001,
-        0x0e9804bd,
-        0x010f9800,
-        0x015021f5,
-        0xbb002fbb,
-        0x0e98003f,
-        0x020f9801,
-        0x015021f5,
-        0xfd050e98,
-        0x2ebb00ef,
-        0x003ebb00,
-        0xf10235b6,
-        0xf0d30007,
-        0x03d00103,
-        0xb604bd00,
-        0x35b60825,
-        0x0120b606,
-        0xb60130b6,
-        0x34b60824,
-        0x022fb908,
-        0x02d321f5,
-        0xf1003fbb,
-        0xf0010007,
-        0x03d00203,
-        0xbd04bd00,
-        0x1f29f024,
-        0x080007f1,
-        0xd00203f0,
         0x04bd0002,
-/* 0x0498: main */
-        0xf40031f4,
-        0xd7f00028,
-        0x3921f41c,
-        0xb0f401f4,
-        0x18f404e4,
-        0x0181fe1e,
-        0xbd0627f0,
-        0x0412fd20,
-        0xfd01e4b6,
-        0x18fe051e,
-        0x8d21f500,
-        0xd30ef405,
-/* 0x04c8: main_not_ctx_xfer */
-        0xf010ef94,
-        0x21f501f5,
-        0x0ef4037e,
-/* 0x04d5: ih */
-        0xfe80f9c6,
-        0x80f90188,
-        0xa0f990f9,
-        0xd0f9b0f9,
-        0xf0f9e0f9,
-        0xa7f104bd,
-        0xa3f00200,
-        0x00aacf00,
-        0xf404abc4,
-        0xd7f02c0b,
-        0x00e7f11c,
-        0x00e3f01a,
-        0xf100eecf,
-        0xf01900f7,
-        0xffcf00f3,
-        0x0421f400,
-        0xf101e7f0,
-        0xf01d0007,
-        0x0ed00003,
-/* 0x0523: ih_no_fifo */
+        0xf11031f4,
+        0xf0820027,
+        0x22cf0123,
+        0x0137f000,
+        0xbb1f24f0,
+        0x32b60432,
+        0x05028001,
+        0xf1060380,
+        0xf0860027,
+        0x22cf0123,
+        0x04028000,
+        0x010027f1,
+        0xcf0223f0,
+        0x34bd0022,
+        0xf1082595,
+        0xf0c00007,
+        0x05d00103,
         0xf104bd00,
-        0xf0010007,
-        0x0ad00003,
-        0xfc04bd00,
-        0xfce0fcf0,
-        0xfcb0fcd0,
-        0xfc90fca0,
-        0x0088fe80,
-        0x32f480fc,
-/* 0x0547: hub_barrier_done */
-        0xf001f800,
-        0x0e9801f7,
-        0x04febb04,
-        0xf102ffb9,
-        0xf09418e7,
-        0x21f440e3,
-/* 0x055f: ctx_redswitch */
-        0xf000f89d,
-        0x07f120f7,
-        0x03f08500,
-        0x000fd001,
-        0xe7f004bd,
-/* 0x0571: ctx_redswitch_delay */
-        0x01e2b608,
-        0xf1fd1bf4,
-        0xf10800f5,
-        0xf10200f5,
+        0xf0c10007,
+        0x05d00103,
+        0x9804bd00,
+        0x0f98000e,
+        0x5021f501,
+        0x002fbb01,
+        0x98003fbb,
+        0x0f98010e,
+        0x5021f502,
+        0x050e9801,
+        0xbb00effd,
+        0x3ebb002e,
+        0x0235b600,
+        0xd30007f1,
+        0xd00103f0,
+        0x04bd0003,
+        0xb60825b6,
+        0x20b60635,
+        0x0130b601,
+        0xb60824b6,
+        0x2fb90834,
+        0xd321f502,
+        0x003fbb02,
+        0x010007f1,
+        0xd00203f0,
+        0x04bd0003,
+        0x29f024bd,
+        0x0007f11f,
+        0x0203f008,
+        0xbd0002d0,
+/* 0x04a9: main */
+        0x0031f404,
+        0xf00028f4,
+        0x21f41cd7,
+        0xf401f439,
+        0xf404e4b0,
+        0x81fe1e18,
+        0x0627f001,
+        0x12fd20bd,
+        0x01e4b604,
+        0xfe051efd,
+        0x21f50018,
+        0x0ef4059e,
+/* 0x04d9: main_not_ctx_xfer */
+        0x10ef94d3,
+        0xf501f5f0,
+        0xf4037e21,
+/* 0x04e6: ih */
+        0x80f9c60e,
+        0xf90188fe,
+        0xf990f980,
+        0xf9b0f9a0,
+        0xf9e0f9d0,
+        0xf104bdf0,
+        0xf00200a7,
+        0xaacf00a3,
+        0x04abc400,
+        0xf02c0bf4,
+        0xe7f11cd7,
+        0xe3f01a00,
+        0x00eecf00,
+        0x1900f7f1,
+        0xcf00f3f0,
+        0x21f400ff,
+        0x01e7f004,
+        0x1d0007f1,
+        0xd00003f0,
+        0x04bd000e,
+/* 0x0534: ih_no_fifo */
+        0x010007f1,
+        0xd00003f0,
+        0x04bd000a,
+        0xe0fcf0fc,
+        0xb0fcd0fc,
+        0x90fca0fc,
+        0x88fe80fc,
+        0xf480fc00,
+        0x01f80032,
+/* 0x0558: hub_barrier_done */
+        0x9801f7f0,
+        0xfebb040e,
+        0x02ffb904,
+        0x9418e7f1,
+        0xf440e3f0,
+        0x00f89d21,
+/* 0x0570: ctx_redswitch */
+        0xf120f7f0,
         0xf0850007,
         0x0fd00103,
-        0xf804bd00,
-/* 0x058d: ctx_xfer */
-        0x0007f100,
-        0x0203f081,
-        0xbd000fd0,
-        0x0711f404,
-        0x055f21f5,
-/* 0x05a0: ctx_xfer_not_load */
-        0x026a21f5,
-        0x07f124bd,
-        0x03f047fc,
-        0x0002d002,
-        0x2cf004bd,
-        0x0320b601,
-        0x4afc07f1,
-        0xd00203f0,
-        0x04bd0002,
+        0xf004bd00,
+/* 0x0582: ctx_redswitch_delay */
+        0xe2b608e7,
+        0xfd1bf401,
+        0x0800f5f1,
+        0x0200f5f1,
+        0x850007f1,
+        0xd00103f0,
+        0x04bd000f,
+/* 0x059e: ctx_xfer */
+        0x07f100f8,
+        0x03f08100,
+        0x000fd002,
+        0x11f404bd,
+        0x7021f507,
+/* 0x05b1: ctx_xfer_not_load */
+        0x6a21f505,
+        0xf124bd02,
+        0xf047fc07,
+        0x02d00203,
+        0xf004bd00,
+        0x20b6012c,
+        0xfc07f103,
+        0x0203f04a,
+        0xbd0002d0,
+        0x01acf004,
+        0xf102a5f0,
+        0xf00000b7,
+        0x0c9850b3,
+        0x0fc4b604,
+        0x9800bcbb,
+        0x0d98000c,
+        0x00e7f001,
+        0x016f21f5,
         0xf001acf0,
-        0xb7f102a5,
-        0xb3f00000,
+        0xb7f104a5,
+        0xb3f04000,
         0x040c9850,
         0xbb0fc4b6,
         0x0c9800bc,
-        0x010d9800,
-        0xf500e7f0,
-        0xf0016f21,
-        0xa5f001ac,
-        0x00b7f104,
-        0x50b3f040,
-        0xb6040c98,
-        0xbcbb0fc4,
-        0x010c9800,
-        0x98020d98,
-        0xe7f1060f,
-        0x21f50800,
-        0x21f5016f,
-        0x01f4025e,
-        0x0712f406,
-/* 0x0618: ctx_xfer_post */
-        0x027f21f5,
-/* 0x061c: ctx_xfer_done */
-        0x054721f5,
-        0x000000f8,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
+        0x020d9801,
+        0xf1060f98,
+        0xf50800e7,
+        0xf5016f21,
+        0xf4025e21,
+        0x12f40601,
+/* 0x0629: ctx_xfer_post */
+        0x7f21f507,
+/* 0x062d: ctx_xfer_done */
+        0x5821f502,
+        0x0000f805,
         0x00000000,
         0x00000000,
         0x00000000,
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvd7.fuc.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvd7.fuc.h
index 84dd32db28a0..d1504a4059c6 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvd7.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvd7.fuc.h
@@ -196,7 +196,7 @@ uint32_t nvd7_grgpc_code[] = {
         0x1fb4f000,
         0xf410b4b0,
         0xa7f0f01b,
-        0xd021f402,
+        0xd021f405,
 /* 0x0223: mmctx_stop */
         0xc82b0ef4,
         0xb4b600ab,
@@ -304,212 +304,212 @@ uint32_t nvd7_grgpc_code[] = {
         0x21f440e3,
         0xf8e0fc9d,
 /* 0x03a1: init */
-        0xfe04bd00,
-        0x27f00004,
-        0x0007f102,
-        0x0003f012,
-        0xbd0002d0,
-        0x1f17f104,
-        0x0010fe05,
-        0x070007f1,
+        0xf104bd00,
+        0xf0420017,
+        0x11cf0013,
+        0x0911e700,
+        0x0814b601,
+        0xf00014fe,
+        0x07f10227,
+        0x03f01200,
+        0x0002d000,
+        0x17f104bd,
+        0x10fe0530,
+        0x0007f100,
+        0x0003f007,
+        0xbd0000d0,
+        0x0427f004,
+        0x040007f1,
         0xd00003f0,
-        0x04bd0000,
-        0xf10427f0,
-        0xf0040007,
-        0x02d00003,
+        0x04bd0002,
+        0xf11031f4,
+        0xf0820027,
+        0x22cf0123,
+        0x0137f000,
+        0xbb1f24f0,
+        0x32b60432,
+        0x05028001,
+        0xf1060380,
+        0xf0860027,
+        0x22cf0123,
+        0x04028000,
+        0x0c30e7f1,
+        0xbd50e3f0,
+        0xbd34bd24,
+/* 0x0421: init_unk_loop */
+        0x6821f444,
+        0xf400f6b0,
+        0xf7f00f0b,
+        0x04f2bb01,
+        0xb6054ffd,
+/* 0x0436: init_unk_next */
+        0x20b60130,
+        0x04e0b601,
+        0xf40126b0,
+/* 0x0442: init_unk_done */
+        0x0380e21b,
+        0x08048007,
+        0x010027f1,
+        0xcf0223f0,
+        0x34bd0022,
+        0xf1082595,
+        0xf0c00007,
+        0x05d00103,
+        0xf104bd00,
+        0xf0c10007,
+        0x05d00103,
+        0x9804bd00,
+        0x0f98000e,
+        0x5021f501,
+        0x002fbb01,
+        0x98003fbb,
+        0x0f98010e,
+        0x5021f502,
+        0x050e9801,
+        0xbb00effd,
+        0x3ebb002e,
+        0x020e9800,
+        0xf5030f98,
+        0x98015021,
+        0xeffd070e,
+        0x002ebb00,
+        0xb6003ebb,
+        0x07f10235,
+        0x03f0d300,
+        0x0003d001,
+        0x25b604bd,
+        0x0635b608,
+        0xb60120b6,
+        0x24b60130,
+        0x0834b608,
+        0xf5022fb9,
+        0xbb02d321,
+        0x07f1003f,
+        0x03f00100,
+        0x0003d002,
+        0x24bd04bd,
+        0xf11f29f0,
+        0xf0080007,
+        0x02d00203,
+/* 0x04f3: main */
         0xf404bd00,
-        0x27f11031,
-        0x23f08200,
-        0x0022cf01,
-        0xf00137f0,
-        0x32bb1f24,
-        0x0132b604,
-        0x80050280,
-        0x27f10603,
-        0x23f08600,
-        0x0022cf01,
-        0xf1040280,
-        0xf00c30e7,
-        0x24bd50e3,
-        0x44bd34bd,
-/* 0x0410: init_unk_loop */
-        0xb06821f4,
-        0x0bf400f6,
-        0x01f7f00f,
-        0xfd04f2bb,
-        0x30b6054f,
-/* 0x0425: init_unk_next */
-        0x0120b601,
-        0xb004e0b6,
-        0x1bf40126,
-/* 0x0431: init_unk_done */
-        0x070380e2,
-        0xf1080480,
-        0xf0010027,
-        0x22cf0223,
-        0x9534bd00,
-        0x07f10825,
-        0x03f0c000,
-        0x0005d001,
+        0x28f40031,
+        0x24d7f000,
+        0xf43921f4,
+        0xe4b0f401,
+        0x1e18f404,
+        0xf00181fe,
+        0x20bd0627,
+        0xb60412fd,
+        0x1efd01e4,
+        0x0018fe05,
+        0x05e821f5,
+/* 0x0523: main_not_ctx_xfer */
+        0x94d30ef4,
+        0xf5f010ef,
+        0x7e21f501,
+        0xc60ef403,
+/* 0x0530: ih */
+        0x88fe80f9,
+        0xf980f901,
+        0xf9a0f990,
+        0xf9d0f9b0,
+        0xbdf0f9e0,
+        0x00a7f104,
+        0x00a3f002,
+        0xc400aacf,
+        0x0bf404ab,
+        0x24d7f02c,
+        0x1a00e7f1,
+        0xcf00e3f0,
+        0xf7f100ee,
+        0xf3f01900,
+        0x00ffcf00,
+        0xf00421f4,
+        0x07f101e7,
+        0x03f01d00,
+        0x000ed000,
+/* 0x057e: ih_no_fifo */
         0x07f104bd,
-        0x03f0c100,
-        0x0005d001,
-        0x0e9804bd,
-        0x010f9800,
-        0x015021f5,
-        0xbb002fbb,
-        0x0e98003f,
-        0x020f9801,
-        0x015021f5,
-        0xfd050e98,
-        0x2ebb00ef,
-        0x003ebb00,
-        0x98020e98,
-        0x21f5030f,
-        0x0e980150,
-        0x00effd07,
-        0xbb002ebb,
-        0x35b6003e,
-        0x0007f102,
-        0x0103f0d3,
-        0xbd0003d0,
-        0x0825b604,
-        0xb60635b6,
-        0x30b60120,
-        0x0824b601,
-        0xb90834b6,
-        0x21f5022f,
-        0x3fbb02d3,
-        0x0007f100,
-        0x0203f001,
-        0xbd0003d0,
-        0xf024bd04,
-        0x07f11f29,
-        0x03f00800,
-        0x0002d002,
-/* 0x04e2: main */
-        0x31f404bd,
-        0x0028f400,
-        0xf424d7f0,
-        0x01f43921,
-        0x04e4b0f4,
-        0xfe1e18f4,
-        0x27f00181,
-        0xfd20bd06,
-        0xe4b60412,
-        0x051efd01,
-        0xf50018fe,
-        0xf405d721,
-/* 0x0512: main_not_ctx_xfer */
-        0xef94d30e,
-        0x01f5f010,
-        0x037e21f5,
-/* 0x051f: ih */
-        0xf9c60ef4,
-        0x0188fe80,
-        0x90f980f9,
-        0xb0f9a0f9,
-        0xe0f9d0f9,
-        0x04bdf0f9,
-        0x0200a7f1,
-        0xcf00a3f0,
-        0xabc400aa,
-        0x2c0bf404,
-        0xf124d7f0,
-        0xf01a00e7,
-        0xeecf00e3,
-        0x00f7f100,
-        0x00f3f019,
-        0xf400ffcf,
-        0xe7f00421,
-        0x0007f101,
-        0x0003f01d,
-        0xbd000ed0,
-/* 0x056d: ih_no_fifo */
-        0x0007f104,
-        0x0003f001,
-        0xbd000ad0,
-        0xfcf0fc04,
-        0xfcd0fce0,
-        0xfca0fcb0,
-        0xfe80fc90,
-        0x80fc0088,
-        0xf80032f4,
-/* 0x0591: hub_barrier_done */
-        0x01f7f001,
-        0xbb040e98,
-        0xffb904fe,
-        0x18e7f102,
-        0x40e3f094,
-        0xf89d21f4,
-/* 0x05a9: ctx_redswitch */
-        0x20f7f000,
-        0x850007f1,
-        0xd00103f0,
-        0x04bd000f,
-/* 0x05bb: ctx_redswitch_delay */
-        0xb608e7f0,
-        0x1bf401e2,
-        0x00f5f1fd,
-        0x00f5f108,
-        0x0007f102,
+        0x03f00100,
+        0x000ad000,
+        0xf0fc04bd,
+        0xd0fce0fc,
+        0xa0fcb0fc,
+        0x80fc90fc,
+        0xfc0088fe,
+        0x0032f480,
+/* 0x05a2: hub_barrier_done */
+        0xf7f001f8,
+        0x040e9801,
+        0xb904febb,
+        0xe7f102ff,
+        0xe3f09418,
+        0x9d21f440,
+/* 0x05ba: ctx_redswitch */
+        0xf7f000f8,
+        0x0007f120,
         0x0103f085,
         0xbd000fd0,
-/* 0x05d7: ctx_xfer */
-        0xf100f804,
-        0xf0810007,
-        0x0fd00203,
-        0xf404bd00,
-        0x21f50711,
-/* 0x05ea: ctx_xfer_not_load */
-        0x21f505a9,
-        0x24bd026a,
-        0x47fc07f1,
+        0x08e7f004,
+/* 0x05cc: ctx_redswitch_delay */
+        0xf401e2b6,
+        0xf5f1fd1b,
+        0xf5f10800,
+        0x07f10200,
+        0x03f08500,
+        0x000fd001,
+        0x00f804bd,
+/* 0x05e8: ctx_xfer */
+        0x810007f1,
         0xd00203f0,
-        0x04bd0002,
-        0xb6012cf0,
-        0x07f10320,
-        0x03f04afc,
-        0x0002d002,
-        0xacf004bd,
-        0x02a5f001,
-        0x0000b7f1,
-        0x9850b3f0,
-        0xc4b6040c,
-        0x00bcbb0f,
-        0x98000c98,
-        0xe7f0010d,
-        0x6f21f500,
-        0x01acf001,
-        0x4000b7f1,
+        0x04bd000f,
+        0xf50711f4,
+/* 0x05fb: ctx_xfer_not_load */
+        0xf505ba21,
+        0xbd026a21,
+        0xfc07f124,
+        0x0203f047,
+        0xbd0002d0,
+        0x012cf004,
+        0xf10320b6,
+        0xf04afc07,
+        0x02d00203,
+        0xf004bd00,
+        0xa5f001ac,
+        0x00b7f102,
+        0x50b3f000,
+        0xb6040c98,
+        0xbcbb0fc4,
+        0x000c9800,
+        0xf0010d98,
+        0x21f500e7,
+        0xacf0016f,
+        0x00b7f101,
+        0x50b3f040,
+        0xb6040c98,
+        0xbcbb0fc4,
+        0x010c9800,
+        0x98020d98,
+        0xe7f1060f,
+        0x21f50800,
+        0xacf0016f,
+        0x04a5f001,
+        0x3000b7f1,
         0x9850b3f0,
         0xc4b6040c,
         0x00bcbb0f,
-        0x98010c98,
-        0x0f98020d,
-        0x00e7f106,
-        0x6f21f508,
-        0x01acf001,
-        0xf104a5f0,
-        0xf03000b7,
-        0x0c9850b3,
-        0x0fc4b604,
-        0x9800bcbb,
-        0x0d98020c,
-        0x080f9803,
-        0x0200e7f1,
-        0x016f21f5,
-        0x025e21f5,
-        0xf40601f4,
-/* 0x0686: ctx_xfer_post */
-        0x21f50712,
-/* 0x068a: ctx_xfer_done */
-        0x21f5027f,
-        0x00f80591,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
+        0x98020c98,
+        0x0f98030d,
+        0x00e7f108,
+        0x6f21f502,
+        0x5e21f501,
+        0x0601f402,
+/* 0x0697: ctx_xfer_post */
+        0xf50712f4,
+/* 0x069b: ctx_xfer_done */
+        0xf5027f21,
+        0xf805a221,
         0x00000000,
         0x00000000,
         0x00000000,
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnve0.fuc.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnve0.fuc.h
index b6da800ee9c2..855b220378f9 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnve0.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnve0.fuc.h
@@ -196,7 +196,7 @@ uint32_t nve0_grgpc_code[] = {
         0x1fb4f000,
         0xf410b4b0,
         0xa7f0f01b,
-        0xd021f402,
+        0xd021f405,
 /* 0x0223: mmctx_stop */
         0xc82b0ef4,
         0xb4b600ab,
@@ -304,212 +304,212 @@ uint32_t nve0_grgpc_code[] = {
         0x21f440e3,
         0xf8e0fc9d,
 /* 0x03a1: init */
-        0xfe04bd00,
-        0x27f00004,
-        0x0007f102,
-        0x0003f012,
-        0xbd0002d0,
-        0x1f17f104,
-        0x0010fe05,
-        0x070007f1,
+        0xf104bd00,
+        0xf0420017,
+        0x11cf0013,
+        0x0911e700,
+        0x0814b601,
+        0xf00014fe,
+        0x07f10227,
+        0x03f01200,
+        0x0002d000,
+        0x17f104bd,
+        0x10fe0530,
+        0x0007f100,
+        0x0003f007,
+        0xbd0000d0,
+        0x0427f004,
+        0x040007f1,
         0xd00003f0,
-        0x04bd0000,
-        0xf10427f0,
-        0xf0040007,
-        0x02d00003,
+        0x04bd0002,
+        0xf11031f4,
+        0xf0820027,
+        0x22cf0123,
+        0x0137f000,
+        0xbb1f24f0,
+        0x32b60432,
+        0x05028001,
+        0xf1060380,
+        0xf0860027,
+        0x22cf0123,
+        0x04028000,
+        0x0c30e7f1,
+        0xbd50e3f0,
+        0xbd34bd24,
+/* 0x0421: init_unk_loop */
+        0x6821f444,
+        0xf400f6b0,
+        0xf7f00f0b,
+        0x04f2bb01,
+        0xb6054ffd,
+/* 0x0436: init_unk_next */
+        0x20b60130,
+        0x04e0b601,
+        0xf40126b0,
+/* 0x0442: init_unk_done */
+        0x0380e21b,
+        0x08048007,
+        0x010027f1,
+        0xcf0223f0,
+        0x34bd0022,
+        0xf1082595,
+        0xf0c00007,
+        0x05d00103,
+        0xf104bd00,
+        0xf0c10007,
+        0x05d00103,
+        0x9804bd00,
+        0x0f98000e,
+        0x5021f501,
+        0x002fbb01,
+        0x98003fbb,
+        0x0f98010e,
+        0x5021f502,
+        0x050e9801,
+        0xbb00effd,
+        0x3ebb002e,
+        0x020e9800,
+        0xf5030f98,
+        0x98015021,
+        0xeffd070e,
+        0x002ebb00,
+        0xb6003ebb,
+        0x07f10235,
+        0x03f0d300,
+        0x0003d001,
+        0x25b604bd,
+        0x0635b608,
+        0xb60120b6,
+        0x24b60130,
+        0x0834b608,
+        0xf5022fb9,
+        0xbb02d321,
+        0x07f1003f,
+        0x03f00100,
+        0x0003d002,
+        0x24bd04bd,
+        0xf11f29f0,
+        0xf0080007,
+        0x02d00203,
+/* 0x04f3: main */
         0xf404bd00,
-        0x27f11031,
-        0x23f08200,
-        0x0022cf01,
-        0xf00137f0,
-        0x32bb1f24,
-        0x0132b604,
-        0x80050280,
-        0x27f10603,
-        0x23f08600,
-        0x0022cf01,
-        0xf1040280,
-        0xf00c30e7,
-        0x24bd50e3,
-        0x44bd34bd,
-/* 0x0410: init_unk_loop */
-        0xb06821f4,
-        0x0bf400f6,
-        0x01f7f00f,
-        0xfd04f2bb,
-        0x30b6054f,
-/* 0x0425: init_unk_next */
-        0x0120b601,
-        0xb004e0b6,
-        0x1bf40126,
-/* 0x0431: init_unk_done */
-        0x070380e2,
-        0xf1080480,
-        0xf0010027,
-        0x22cf0223,
-        0x9534bd00,
-        0x07f10825,
-        0x03f0c000,
-        0x0005d001,
+        0x28f40031,
+        0x24d7f000,
+        0xf43921f4,
+        0xe4b0f401,
+        0x1e18f404,
+        0xf00181fe,
+        0x20bd0627,
+        0xb60412fd,
+        0x1efd01e4,
+        0x0018fe05,
+        0x05e821f5,
+/* 0x0523: main_not_ctx_xfer */
+        0x94d30ef4,
+        0xf5f010ef,
+        0x7e21f501,
+        0xc60ef403,
+/* 0x0530: ih */
+        0x88fe80f9,
+        0xf980f901,
+        0xf9a0f990,
+        0xf9d0f9b0,
+        0xbdf0f9e0,
+        0x00a7f104,
+        0x00a3f002,
+        0xc400aacf,
+        0x0bf404ab,
+        0x24d7f02c,
+        0x1a00e7f1,
+        0xcf00e3f0,
+        0xf7f100ee,
+        0xf3f01900,
+        0x00ffcf00,
+        0xf00421f4,
+        0x07f101e7,
+        0x03f01d00,
+        0x000ed000,
+/* 0x057e: ih_no_fifo */
         0x07f104bd,
-        0x03f0c100,
-        0x0005d001,
-        0x0e9804bd,
-        0x010f9800,
-        0x015021f5,
-        0xbb002fbb,
-        0x0e98003f,
-        0x020f9801,
-        0x015021f5,
-        0xfd050e98,
-        0x2ebb00ef,
-        0x003ebb00,
-        0x98020e98,
-        0x21f5030f,
-        0x0e980150,
-        0x00effd07,
-        0xbb002ebb,
-        0x35b6003e,
-        0x0007f102,
-        0x0103f0d3,
-        0xbd0003d0,
-        0x0825b604,
-        0xb60635b6,
-        0x30b60120,
-        0x0824b601,
-        0xb90834b6,
-        0x21f5022f,
-        0x3fbb02d3,
-        0x0007f100,
-        0x0203f001,
-        0xbd0003d0,
-        0xf024bd04,
-        0x07f11f29,
-        0x03f00800,
-        0x0002d002,
-/* 0x04e2: main */
-        0x31f404bd,
-        0x0028f400,
-        0xf424d7f0,
-        0x01f43921,
-        0x04e4b0f4,
-        0xfe1e18f4,
-        0x27f00181,
-        0xfd20bd06,
-        0xe4b60412,
-        0x051efd01,
-        0xf50018fe,
-        0xf405d721,
-/* 0x0512: main_not_ctx_xfer */
-        0xef94d30e,
-        0x01f5f010,
-        0x037e21f5,
-/* 0x051f: ih */
-        0xf9c60ef4,
-        0x0188fe80,
-        0x90f980f9,
-        0xb0f9a0f9,
-        0xe0f9d0f9,
-        0x04bdf0f9,
-        0x0200a7f1,
-        0xcf00a3f0,
-        0xabc400aa,
-        0x2c0bf404,
-        0xf124d7f0,
-        0xf01a00e7,
-        0xeecf00e3,
-        0x00f7f100,
-        0x00f3f019,
-        0xf400ffcf,
-        0xe7f00421,
-        0x0007f101,
-        0x0003f01d,
-        0xbd000ed0,
-/* 0x056d: ih_no_fifo */
-        0x0007f104,
-        0x0003f001,
-        0xbd000ad0,
-        0xfcf0fc04,
-        0xfcd0fce0,
-        0xfca0fcb0,
-        0xfe80fc90,
-        0x80fc0088,
-        0xf80032f4,
-/* 0x0591: hub_barrier_done */
-        0x01f7f001,
-        0xbb040e98,
-        0xffb904fe,
-        0x18e7f102,
-        0x40e3f094,
-        0xf89d21f4,
-/* 0x05a9: ctx_redswitch */
-        0x20f7f000,
-        0x850007f1,
-        0xd00103f0,
-        0x04bd000f,
-/* 0x05bb: ctx_redswitch_delay */
-        0xb608e7f0,
-        0x1bf401e2,
-        0x00f5f1fd,
-        0x00f5f108,
-        0x0007f102,
+        0x03f00100,
+        0x000ad000,
+        0xf0fc04bd,
+        0xd0fce0fc,
+        0xa0fcb0fc,
+        0x80fc90fc,
+        0xfc0088fe,
+        0x0032f480,
+/* 0x05a2: hub_barrier_done */
+        0xf7f001f8,
+        0x040e9801,
+        0xb904febb,
+        0xe7f102ff,
+        0xe3f09418,
+        0x9d21f440,
+/* 0x05ba: ctx_redswitch */
+        0xf7f000f8,
+        0x0007f120,
         0x0103f085,
         0xbd000fd0,
-/* 0x05d7: ctx_xfer */
-        0xf100f804,
-        0xf0810007,
-        0x0fd00203,
-        0xf404bd00,
-        0x21f50711,
-/* 0x05ea: ctx_xfer_not_load */
-        0x21f505a9,
-        0x24bd026a,
-        0x47fc07f1,
+        0x08e7f004,
+/* 0x05cc: ctx_redswitch_delay */
+        0xf401e2b6,
+        0xf5f1fd1b,
+        0xf5f10800,
+        0x07f10200,
+        0x03f08500,
+        0x000fd001,
+        0x00f804bd,
+/* 0x05e8: ctx_xfer */
+        0x810007f1,
         0xd00203f0,
-        0x04bd0002,
-        0xb6012cf0,
-        0x07f10320,
-        0x03f04afc,
-        0x0002d002,
-        0xacf004bd,
-        0x02a5f001,
-        0x0000b7f1,
-        0x9850b3f0,
-        0xc4b6040c,
-        0x00bcbb0f,
-        0x98000c98,
-        0xe7f0010d,
-        0x6f21f500,
-        0x01acf001,
-        0x4000b7f1,
+        0x04bd000f,
+        0xf50711f4,
+/* 0x05fb: ctx_xfer_not_load */
+        0xf505ba21,
+        0xbd026a21,
+        0xfc07f124,
+        0x0203f047,
+        0xbd0002d0,
+        0x012cf004,
+        0xf10320b6,
+        0xf04afc07,
+        0x02d00203,
+        0xf004bd00,
+        0xa5f001ac,
+        0x00b7f102,
+        0x50b3f000,
+        0xb6040c98,
+        0xbcbb0fc4,
+        0x000c9800,
+        0xf0010d98,
+        0x21f500e7,
+        0xacf0016f,
+        0x00b7f101,
+        0x50b3f040,
+        0xb6040c98,
+        0xbcbb0fc4,
+        0x010c9800,
+        0x98020d98,
+        0xe7f1060f,
+        0x21f50800,
+        0xacf0016f,
+        0x04a5f001,
+        0x3000b7f1,
         0x9850b3f0,
         0xc4b6040c,
         0x00bcbb0f,
-        0x98010c98,
-        0x0f98020d,
-        0x00e7f106,
-        0x6f21f508,
-        0x01acf001,
-        0xf104a5f0,
-        0xf03000b7,
-        0x0c9850b3,
-        0x0fc4b604,
-        0x9800bcbb,
-        0x0d98020c,
-        0x080f9803,
-        0x0200e7f1,
-        0x016f21f5,
-        0x025e21f5,
-        0xf40601f4,
-/* 0x0686: ctx_xfer_post */
-        0x21f50712,
-/* 0x068a: ctx_xfer_done */
-        0x21f5027f,
-        0x00f80591,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
+        0x98020c98,
+        0x0f98030d,
+        0x00e7f108,
+        0x6f21f502,
+        0x5e21f501,
+        0x0601f402,
+/* 0x0697: ctx_xfer_post */
+        0xf50712f4,
+/* 0x069b: ctx_xfer_done */
+        0xf5027f21,
+        0xf805a221,
         0x00000000,
         0x00000000,
         0x00000000,
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvf0.fuc.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvf0.fuc.h
index 6316ebaf5d9a..1b803197d28b 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvf0.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/gpcnvf0.fuc.h
@@ -196,7 +196,7 @@ uint32_t nvf0_grgpc_code[] = {
         0x1fb4f000,
         0xf410b4b0,
         0xa7f0f01b,
-        0xd021f402,
+        0xd021f405,
 /* 0x0223: mmctx_stop */
         0xc82b0ef4,
         0xb4b600ab,
@@ -304,212 +304,212 @@ uint32_t nvf0_grgpc_code[] = {
         0x21f440e3,
         0xf8e0fc9d,
 /* 0x03a1: init */
-        0xfe04bd00,
-        0x27f00004,
-        0x0007f102,
-        0x0003f012,
-        0xbd0002d0,
-        0x1f17f104,
-        0x0010fe05,
-        0x070007f1,
+        0xf104bd00,
+        0xf0420017,
+        0x11cf0013,
+        0x0911e700,
+        0x0814b601,
+        0xf00014fe,
+        0x07f10227,
+        0x03f01200,
+        0x0002d000,
+        0x17f104bd,
+        0x10fe0530,
+        0x0007f100,
+        0x0003f007,
+        0xbd0000d0,
+        0x0427f004,
+        0x040007f1,
         0xd00003f0,
-        0x04bd0000,
-        0xf10427f0,
-        0xf0040007,
-        0x02d00003,
+        0x04bd0002,
+        0xf11031f4,
+        0xf0820027,
+        0x22cf0123,
+        0x0137f000,
+        0xbb1f24f0,
+        0x32b60432,
+        0x05028001,
+        0xf1060380,
+        0xf0860027,
+        0x22cf0123,
+        0x04028000,
+        0x0c30e7f1,
+        0xbd50e3f0,
+        0xbd34bd24,
+/* 0x0421: init_unk_loop */
+        0x6821f444,
+        0xf400f6b0,
+        0xf7f00f0b,
+        0x04f2bb01,
+        0xb6054ffd,
+/* 0x0436: init_unk_next */
+        0x20b60130,
+        0x04e0b601,
+        0xf40226b0,
+/* 0x0442: init_unk_done */
+        0x0380e21b,
+        0x08048007,
+        0x010027f1,
+        0xcf0223f0,
+        0x34bd0022,
+        0xf1082595,
+        0xf0c00007,
+        0x05d00103,
+        0xf104bd00,
+        0xf0c10007,
+        0x05d00103,
+        0x9804bd00,
+        0x0f98000e,
+        0x5021f501,
+        0x002fbb01,
+        0x98003fbb,
+        0x0f98010e,
+        0x5021f502,
+        0x050e9801,
+        0xbb00effd,
+        0x3ebb002e,
+        0x020e9800,
+        0xf5030f98,
+        0x98015021,
+        0xeffd070e,
+        0x002ebb00,
+        0xb6003ebb,
+        0x07f10235,
+        0x03f0d300,
+        0x0003d001,
+        0x25b604bd,
+        0x0635b608,
+        0xb60120b6,
+        0x24b60130,
+        0x0834b608,
+        0xf5022fb9,
+        0xbb02d321,
+        0x07f1003f,
+        0x03f00100,
+        0x0003d002,
+        0x24bd04bd,
+        0xf11f29f0,
+        0xf0300007,
+        0x02d00203,
+/* 0x04f3: main */
         0xf404bd00,
-        0x27f11031,
-        0x23f08200,
-        0x0022cf01,
-        0xf00137f0,
-        0x32bb1f24,
-        0x0132b604,
-        0x80050280,
-        0x27f10603,
-        0x23f08600,
-        0x0022cf01,
-        0xf1040280,
-        0xf00c30e7,
-        0x24bd50e3,
-        0x44bd34bd,
-/* 0x0410: init_unk_loop */
-        0xb06821f4,
-        0x0bf400f6,
-        0x01f7f00f,
-        0xfd04f2bb,
-        0x30b6054f,
-/* 0x0425: init_unk_next */
-        0x0120b601,
-        0xb004e0b6,
-        0x1bf40226,
-/* 0x0431: init_unk_done */
-        0x070380e2,
-        0xf1080480,
-        0xf0010027,
-        0x22cf0223,
-        0x9534bd00,
-        0x07f10825,
-        0x03f0c000,
-        0x0005d001,
+        0x28f40031,
+        0x24d7f000,
+        0xf43921f4,
+        0xe4b0f401,
+        0x1e18f404,
+        0xf00181fe,
+        0x20bd0627,
+        0xb60412fd,
+        0x1efd01e4,
+        0x0018fe05,
+        0x05e821f5,
+/* 0x0523: main_not_ctx_xfer */
+        0x94d30ef4,
+        0xf5f010ef,
+        0x7e21f501,
+        0xc60ef403,
+/* 0x0530: ih */
+        0x88fe80f9,
+        0xf980f901,
+        0xf9a0f990,
+        0xf9d0f9b0,
+        0xbdf0f9e0,
+        0x00a7f104,
+        0x00a3f002,
+        0xc400aacf,
+        0x0bf404ab,
+        0x24d7f02c,
+        0x1a00e7f1,
+        0xcf00e3f0,
+        0xf7f100ee,
+        0xf3f01900,
+        0x00ffcf00,
+        0xf00421f4,
+        0x07f101e7,
+        0x03f01d00,
+        0x000ed000,
+/* 0x057e: ih_no_fifo */
         0x07f104bd,
-        0x03f0c100,
-        0x0005d001,
-        0x0e9804bd,
-        0x010f9800,
-        0x015021f5,
-        0xbb002fbb,
-        0x0e98003f,
-        0x020f9801,
-        0x015021f5,
-        0xfd050e98,
-        0x2ebb00ef,
-        0x003ebb00,
-        0x98020e98,
-        0x21f5030f,
-        0x0e980150,
-        0x00effd07,
-        0xbb002ebb,
-        0x35b6003e,
-        0x0007f102,
-        0x0103f0d3,
-        0xbd0003d0,
-        0x0825b604,
-        0xb60635b6,
-        0x30b60120,
-        0x0824b601,
-        0xb90834b6,
-        0x21f5022f,
-        0x3fbb02d3,
-        0x0007f100,
-        0x0203f001,
-        0xbd0003d0,
-        0xf024bd04,
-        0x07f11f29,
-        0x03f03000,
-        0x0002d002,
-/* 0x04e2: main */
-        0x31f404bd,
-        0x0028f400,
-        0xf424d7f0,
-        0x01f43921,
-        0x04e4b0f4,
-        0xfe1e18f4,
-        0x27f00181,
-        0xfd20bd06,
-        0xe4b60412,
-        0x051efd01,
-        0xf50018fe,
-        0xf405d721,
-/* 0x0512: main_not_ctx_xfer */
-        0xef94d30e,
-        0x01f5f010,
-        0x037e21f5,
-/* 0x051f: ih */
-        0xf9c60ef4,
-        0x0188fe80,
-        0x90f980f9,
-        0xb0f9a0f9,
-        0xe0f9d0f9,
-        0x04bdf0f9,
-        0x0200a7f1,
-        0xcf00a3f0,
-        0xabc400aa,
-        0x2c0bf404,
-        0xf124d7f0,
-        0xf01a00e7,
-        0xeecf00e3,
-        0x00f7f100,
-        0x00f3f019,
-        0xf400ffcf,
-        0xe7f00421,
-        0x0007f101,
-        0x0003f01d,
-        0xbd000ed0,
-/* 0x056d: ih_no_fifo */
-        0x0007f104,
-        0x0003f001,
-        0xbd000ad0,
-        0xfcf0fc04,
-        0xfcd0fce0,
-        0xfca0fcb0,
-        0xfe80fc90,
-        0x80fc0088,
-        0xf80032f4,
-/* 0x0591: hub_barrier_done */
-        0x01f7f001,
-        0xbb040e98,
-        0xffb904fe,
-        0x18e7f102,
-        0x40e3f094,
-        0xf89d21f4,
-/* 0x05a9: ctx_redswitch */
-        0x20f7f000,
-        0x850007f1,
-        0xd00103f0,
-        0x04bd000f,
-/* 0x05bb: ctx_redswitch_delay */
-        0xb608e7f0,
-        0x1bf401e2,
-        0x00f5f1fd,
-        0x00f5f108,
-        0x0007f102,
+        0x03f00100,
+        0x000ad000,
+        0xf0fc04bd,
+        0xd0fce0fc,
+        0xa0fcb0fc,
+        0x80fc90fc,
+        0xfc0088fe,
+        0x0032f480,
+/* 0x05a2: hub_barrier_done */
+        0xf7f001f8,
+        0x040e9801,
+        0xb904febb,
+        0xe7f102ff,
+        0xe3f09418,
+        0x9d21f440,
+/* 0x05ba: ctx_redswitch */
+        0xf7f000f8,
+        0x0007f120,
         0x0103f085,
         0xbd000fd0,
-/* 0x05d7: ctx_xfer */
-        0xf100f804,
-        0xf0810007,
-        0x0fd00203,
-        0xf404bd00,
-        0x21f50711,
-/* 0x05ea: ctx_xfer_not_load */
-        0x21f505a9,
-        0x24bd026a,
-        0x47fc07f1,
+        0x08e7f004,
+/* 0x05cc: ctx_redswitch_delay */
+        0xf401e2b6,
+        0xf5f1fd1b,
+        0xf5f10800,
+        0x07f10200,
+        0x03f08500,
+        0x000fd001,
+        0x00f804bd,
+/* 0x05e8: ctx_xfer */
+        0x810007f1,
         0xd00203f0,
-        0x04bd0002,
-        0xb6012cf0,
-        0x07f10320,
-        0x03f04afc,
-        0x0002d002,
-        0xacf004bd,
-        0x02a5f001,
-        0x0000b7f1,
-        0x9850b3f0,
-        0xc4b6040c,
-        0x00bcbb0f,
-        0x98000c98,
-        0xe7f0010d,
-        0x6f21f500,
-        0x01acf001,
-        0x4000b7f1,
+        0x04bd000f,
+        0xf50711f4,
+/* 0x05fb: ctx_xfer_not_load */
+        0xf505ba21,
+        0xbd026a21,
+        0xfc07f124,
+        0x0203f047,
+        0xbd0002d0,
+        0x012cf004,
+        0xf10320b6,
+        0xf04afc07,
+        0x02d00203,
+        0xf004bd00,
+        0xa5f001ac,
+        0x00b7f102,
+        0x50b3f000,
+        0xb6040c98,
+        0xbcbb0fc4,
+        0x000c9800,
+        0xf0010d98,
+        0x21f500e7,
+        0xacf0016f,
+        0x00b7f101,
+        0x50b3f040,
+        0xb6040c98,
+        0xbcbb0fc4,
+        0x010c9800,
+        0x98020d98,
+        0xe7f1060f,
+        0x21f50800,
+        0xacf0016f,
+        0x04a5f001,
+        0x3000b7f1,
         0x9850b3f0,
         0xc4b6040c,
         0x00bcbb0f,
-        0x98010c98,
-        0x0f98020d,
-        0x00e7f106,
-        0x6f21f508,
-        0x01acf001,
-        0xf104a5f0,
-        0xf03000b7,
-        0x0c9850b3,
-        0x0fc4b604,
-        0x9800bcbb,
-        0x0d98020c,
-        0x080f9803,
-        0x0200e7f1,
-        0x016f21f5,
-        0x025e21f5,
-        0xf40601f4,
-/* 0x0686: ctx_xfer_post */
-        0x21f50712,
-/* 0x068a: ctx_xfer_done */
-        0x21f5027f,
-        0x00f80591,
-        0x00000000,
-        0x00000000,
-        0x00000000,
-        0x00000000,
+        0x98020c98,
+        0x0f98030d,
+        0x00e7f108,
+        0x6f21f502,
+        0x5e21f501,
+        0x0601f402,
+/* 0x0697: ctx_xfer_post */
+        0xf50712f4,
+/* 0x069b: ctx_xfer_done */
+        0xf5027f21,
+        0xf805a221,
         0x00000000,
         0x00000000,
         0x00000000,
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubgm107.fuc5 b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubgm107.fuc5
new file mode 100644
index 000000000000..27591b3086a5
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubgm107.fuc5
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#define CHIPSET GK208
+#include "macros.fuc"
+
+.section #gm107_grhub_data
+#define INCLUDE_DATA
+#include "com.fuc"
+#include "hub.fuc"
+#undef INCLUDE_DATA
+
+.section #gm107_grhub_code
+#define INCLUDE_CODE
+bra #init
+#include "com.fuc"
+#include "hub.fuc"
+.align 256
+#undef INCLUDE_CODE
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubgm107.fuc5.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubgm107.fuc5.h
new file mode 100644
index 000000000000..214dd16ec566
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubgm107.fuc5.h
@@ -0,0 +1,916 @@
+uint32_t gm107_grhub_data[] = {
+/* 0x0000: hub_mmio_list_head */
+        0x00000300,
+/* 0x0004: hub_mmio_list_tail */
+        0x00000304,
+/* 0x0008: gpc_count */
+        0x00000000,
+/* 0x000c: rop_count */
+        0x00000000,
+/* 0x0010: cmd_queue */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0058: ctx_current */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0100: chan_data */
+/* 0x0100: chan_mmio_count */
+        0x00000000,
+/* 0x0104: chan_mmio_address */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0200: xfer_data */
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+/* 0x0300: hub_mmio_list_base */
+        0x0417e91c,
+};
+
+uint32_t gm107_grhub_code[] = {
+        0x030e0ef5,
+/* 0x0004: queue_put */
+        0x9800d898,
+        0x86f001d9,
+        0xf489a408,
+        0x020f0b1b,
+        0x0002f87e,
+/* 0x001a: queue_put_next */
+        0x98c400f8,
+        0x0384b607,
+        0xb6008dbb,
+        0x8eb50880,
+        0x018fb500,
+        0xf00190b6,
+        0xd9b50f94,
+/* 0x0037: queue_get */
+        0xf400f801,
+        0xd8980131,
+        0x01d99800,
+        0x0bf489a4,
+        0x0789c421,
+        0xbb0394b6,
+        0x90b6009d,
+        0x009e9808,
+        0xb6019f98,
+        0x84f00180,
+        0x00d8b50f,
+/* 0x0063: queue_get_done */
+        0xf80132f4,
+/* 0x0065: nv_rd32 */
+        0xf0ecb200,
+        0x00801fc9,
+        0x0cf601ca,
+/* 0x0073: nv_rd32_wait */
+        0x8c04bd00,
+        0xcf01ca00,
+        0xccc800cc,
+        0xf61bf41f,
+        0xec7e060a,
+        0x008f0000,
+        0xffcf01cb,
+/* 0x008f: nv_wr32 */
+        0x8000f800,
+        0xf601cc00,
+        0x04bd000f,
+        0xc9f0ecb2,
+        0x1ec9f01f,
+        0x01ca0080,
+        0xbd000cf6,
+/* 0x00a9: nv_wr32_wait */
+        0xca008c04,
+        0x00cccf01,
+        0xf41fccc8,
+        0x00f8f61b,
+/* 0x00b8: wait_donez */
+        0x99f094bd,
+        0x37008000,
+        0x0009f602,
+        0x008004bd,
+        0x0af60206,
+/* 0x00cf: wait_donez_ne */
+        0x8804bd00,
+        0xcf010000,
+        0x8aff0088,
+        0xf61bf488,
+        0x99f094bd,
+        0x17008000,
+        0x0009f602,
+        0x00f804bd,
+/* 0x00ec: wait_doneo */
+        0x99f094bd,
+        0x37008000,
+        0x0009f602,
+        0x008004bd,
+        0x0af60206,
+/* 0x0103: wait_doneo_e */
+        0x8804bd00,
+        0xcf010000,
+        0x8aff0088,
+        0xf60bf488,
+        0x99f094bd,
+        0x17008000,
+        0x0009f602,
+        0x00f804bd,
+/* 0x0120: mmctx_size */
+/* 0x0122: nv_mmctx_size_loop */
+        0xe89894bd,
+        0x1a85b600,
+        0xb60180b6,
+        0x98bb0284,
+        0x04e0b600,
+        0x1bf4efa4,
+        0xf89fb2ec,
+/* 0x013d: mmctx_xfer */
+        0xf094bd00,
+        0x00800199,
+        0x09f60237,
+        0xbd04bd00,
+        0x05bbfd94,
+        0x800f0bf4,
+        0xf601c400,
+        0x04bd000b,
+/* 0x015f: mmctx_base_disabled */
+        0xfd0099f0,
+        0x0bf405ee,
+        0xc6008018,
+        0x000ef601,
+        0x008004bd,
+        0x0ff601c7,
+        0xf004bd00,
+/* 0x017a: mmctx_multi_disabled */
+        0xabc80199,
+        0x10b4b600,
+        0xc80cb9f0,
+        0xe4b601ae,
+        0x05befd11,
+        0x01c50080,
+        0xbd000bf6,
+/* 0x0195: mmctx_exec_loop */
+/* 0x0195: mmctx_wait_free */
+        0xc5008e04,
+        0x00eecf01,
+        0xf41fe4f0,
+        0xce98f60b,
+        0x05e9fd00,
+        0x01c80080,
+        0xbd000ef6,
+        0x04c0b604,
+        0x1bf4cda4,
+        0x02abc8df,
+/* 0x01bf: mmctx_fini_wait */
+        0x8b1c1bf4,
+        0xcf01c500,
+        0xb4f000bb,
+        0x10b4b01f,
+        0x0af31bf4,
+        0x00b87e05,
+        0x250ef400,
+/* 0x01d8: mmctx_stop */
+        0xb600abc8,
+        0xb9f010b4,
+        0x12b9f00c,
+        0x01c50080,
+        0xbd000bf6,
+/* 0x01ed: mmctx_stop_wait */
+        0xc5008b04,
+        0x00bbcf01,
+        0xf412bbc8,
+/* 0x01fa: mmctx_done */
+        0x94bdf61b,
+        0x800199f0,
+        0xf6021700,
+        0x04bd0009,
+/* 0x020a: strand_wait */
+        0xa0f900f8,
+        0xb87e020a,
+        0xa0fc0000,
+/* 0x0216: strand_pre */
+        0x0c0900f8,
+        0x024afc80,
+        0xbd0009f6,
+        0x020a7e04,
+/* 0x0227: strand_post */
+        0x0900f800,
+        0x4afc800d,
+        0x0009f602,
+        0x0a7e04bd,
+        0x00f80002,
+/* 0x0238: strand_set */
+        0xfc800f0c,
+        0x0cf6024f,
+        0x0c04bd00,
+        0x4afc800b,
+        0x000cf602,
+        0xfc8004bd,
+        0x0ef6024f,
+        0x0c04bd00,
+        0x4afc800a,
+        0x000cf602,
+        0x0a7e04bd,
+        0x00f80002,
+/* 0x0268: strand_ctx_init */
+        0x99f094bd,
+        0x37008003,
+        0x0009f602,
+        0x167e04bd,
+        0x030e0002,
+        0x0002387e,
+        0xfc80c4bd,
+        0x0cf60247,
+        0x0c04bd00,
+        0x4afc8001,
+        0x000cf602,
+        0x0a7e04bd,
+        0x0c920002,
+        0x46fc8001,
+        0x000cf602,
+        0x020c04bd,
+        0x024afc80,
+        0xbd000cf6,
+        0x020a7e04,
+        0x02277e00,
+        0x42008800,
+        0x20008902,
+        0x0099cf02,
+/* 0x02c7: ctx_init_strand_loop */
+        0xf608fe95,
+        0x8ef6008e,
+        0x808acf40,
+        0xb606a5b6,
+        0xeabb01a0,
+        0x0480b600,
+        0xf40192b6,
+        0xe4b6e81b,
+        0xf2efbc08,
+        0x99f094bd,
+        0x17008003,
+        0x0009f602,
+        0x00f804bd,
+/* 0x02f8: error */
+        0x02050080,
+        0xbd000ff6,
+        0x80010f04,
+        0xf6030700,
+        0x04bd000f,
+/* 0x030e: init */
+        0x04bd00f8,
+        0x410007fe,
+        0x11cf4200,
+        0x0911e700,
+        0x0814b601,
+        0x020014fe,
+        0x12004002,
+        0xbd0002f6,
+        0x05c94104,
+        0xbd0010fe,
+        0x07004024,
+        0xbd0002f6,
+        0x20034204,
+        0x01010080,
+        0xbd0002f6,
+        0x20044204,
+        0x01010480,
+        0xbd0002f6,
+        0x200b4204,
+        0x01010880,
+        0xbd0002f6,
+        0x200c4204,
+        0x01011c80,
+        0xbd0002f6,
+        0x01039204,
+        0x03090080,
+        0xbd0003f6,
+        0x87044204,
+        0xf6040040,
+        0x04bd0002,
+        0x00400402,
+        0x0002f603,
+        0x31f404bd,
+        0x96048e10,
+        0x00657e40,
+        0xc7feb200,
+        0x01b590f1,
+        0x1ff4f003,
+        0x01020fb5,
+        0x041fbb01,
+        0x800112b6,
+        0xf6010300,
+        0x04bd0001,
+        0x01040080,
+        0xbd0001f6,
+        0x01004104,
+        0x627e020f,
+        0x717e0006,
+        0x100f0006,
+        0x0006b37e,
+        0x98000e98,
+        0x207e010f,
+        0x14950001,
+        0xc0008008,
+        0x0004f601,
+        0x008004bd,
+        0x04f601c1,
+        0xb704bd00,
+        0xbb130030,
+        0xf5b6001f,
+        0xd3008002,
+        0x000ff601,
+        0x15b604bd,
+        0x0110b608,
+        0xb20814b6,
+        0x02687e1f,
+        0x001fbb00,
+        0x84020398,
+/* 0x041f: init_gpc */
+        0xb8502000,
+        0x0008044e,
+        0x8f7e1fb2,
+        0x4eb80000,
+        0xbd00010c,
+        0x008f7ef4,
+        0x044eb800,
+        0x8f7e0001,
+        0x4eb80000,
+        0x0f000100,
+        0x008f7e02,
+        0x004eb800,
+/* 0x044e: init_gpc_wait */
+        0x657e0008,
+        0xffc80000,
+        0xf90bf41f,
+        0x08044eb8,
+        0x00657e00,
+        0x001fbb00,
+        0x800040b7,
+        0xf40132b6,
+        0x000fb41b,
+        0x0006b37e,
+        0x627e000f,
+        0x00800006,
+        0x01f60201,
+        0xbd04bd00,
+        0x1f19f014,
+        0x02300080,
+        0xbd0001f6,
+/* 0x0491: main */
+        0x0031f404,
+        0x0d0028f4,
+        0x00377e10,
+        0xf401f400,
+        0x4001e4b1,
+        0x00c71bf5,
+        0x99f094bd,
+        0x37008004,
+        0x0009f602,
+        0x008104bd,
+        0x11cf02c0,
+        0xc1008200,
+        0x0022cf02,
+        0xf41f13c8,
+        0x23c8770b,
+        0x550bf41f,
+        0x12b220f9,
+        0x99f094bd,
+        0x37008007,
+        0x0009f602,
+        0x32f404bd,
+        0x0231f401,
+        0x0008367e,
+        0x99f094bd,
+        0x17008007,
+        0x0009f602,
+        0x20fc04bd,
+        0x99f094bd,
+        0x37008006,
+        0x0009f602,
+        0x31f404bd,
+        0x08367e01,
+        0xf094bd00,
+        0x00800699,
+        0x09f60217,
+        0xf404bd00,
+/* 0x0522: chsw_prev_no_next */
+        0x20f92f0e,
+        0x32f412b2,
+        0x0232f401,
+        0x0008367e,
+        0x008020fc,
+        0x02f602c0,
+        0xf404bd00,
+/* 0x053e: chsw_no_prev */
+        0x23c8130e,
+        0x0d0bf41f,
+        0xf40131f4,
+        0x367e0232,
+/* 0x054e: chsw_done */
+        0x01020008,
+        0x02c30080,
+        0xbd0002f6,
+        0xf094bd04,
+        0x00800499,
+        0x09f60217,
+        0xf504bd00,
+/* 0x056b: main_not_ctx_switch */
+        0xb0ff2a0e,
+        0x1bf401e4,
+        0x7ef2b20c,
+        0xf40007d6,
+/* 0x057a: main_not_ctx_chan */
+        0xe4b0400e,
+        0x2c1bf402,
+        0x99f094bd,
+        0x37008007,
+        0x0009f602,
+        0x32f404bd,
+        0x0232f401,
+        0x0008367e,
+        0x99f094bd,
+        0x17008007,
+        0x0009f602,
+        0x0ef404bd,
+/* 0x05a9: main_not_ctx_save */
+        0x10ef9411,
+        0x7e01f5f0,
+        0xf50002f8,
+/* 0x05b7: main_done */
+        0xbdfede0e,
+        0x1f29f024,
+        0x02300080,
+        0xbd0002f6,
+        0xcc0ef504,
+/* 0x05c9: ih */
+        0xfe80f9fe,
+        0x80f90188,
+        0xa0f990f9,
+        0xd0f9b0f9,
+        0xf0f9e0f9,
+        0x004a04bd,
+        0x00aacf02,
+        0xf404abc4,
+        0x100d230b,
+        0xcf1a004e,
+        0x004f00ee,
+        0x00ffcf19,
+        0x0000047e,
+        0x0400b0b7,
+        0x0040010e,
+        0x000ef61d,
+/* 0x060a: ih_no_fifo */
+        0xabe404bd,
+        0x0bf40100,
+        0x4e100d0c,
+        0x047e4001,
+/* 0x061a: ih_no_ctxsw */
+        0xabe40000,
+        0x0bf40400,
+        0x01004b10,
+        0x448ebfb2,
+        0x8f7e4001,
+/* 0x062e: ih_no_fwmthd */
+        0x044b0000,
+        0xffb0bd01,
+        0x0bf4b4ab,
+        0x0700800c,
+        0x000bf603,
+/* 0x0642: ih_no_other */
+        0x004004bd,
+        0x000af601,
+        0xf0fc04bd,
+        0xd0fce0fc,
+        0xa0fcb0fc,
+        0x80fc90fc,
+        0xfc0088fe,
+        0x0032f480,
+/* 0x0662: ctx_4170s */
+        0xf5f001f8,
+        0x8effb210,
+        0x7e404170,
+        0xf800008f,
+/* 0x0671: ctx_4170w */
+        0x41708e00,
+        0x00657e40,
+        0xf0ffb200,
+        0x1bf410f4,
+/* 0x0683: ctx_redswitch */
+        0x4e00f8f3,
+        0xe5f00200,
+        0x20e5f040,
+        0x8010e5f0,
+        0xf6018500,
+        0x04bd000e,
+/* 0x069a: ctx_redswitch_delay */
+        0xf2b6080f,
+        0xfd1bf401,
+        0x0400e5f1,
+        0x0100e5f1,
+        0x01850080,
+        0xbd000ef6,
+/* 0x06b3: ctx_86c */
+        0x8000f804,
+        0xf6022300,
+        0x04bd000f,
+        0x148effb2,
+        0x8f7e408a,
+        0xffb20000,
+        0x41a88c8e,
+        0x00008f7e,
+/* 0x06d2: ctx_mem */
+        0x008000f8,
+        0x0ff60284,
+/* 0x06db: ctx_mem_wait */
+        0x8f04bd00,
+        0xcf028400,
+        0xfffd00ff,
+        0xf61bf405,
+/* 0x06ea: ctx_load */
+        0x94bd00f8,
+        0x800599f0,
+        0xf6023700,
+        0x04bd0009,
+        0xb87e0c0a,
+        0xf4bd0000,
+        0x02890080,
+        0xbd000ff6,
+        0xc1008004,
+        0x0002f602,
+        0x008004bd,
+        0x02f60283,
+        0x0f04bd00,
+        0x06d27e07,
+        0xc0008000,
+        0x0002f602,
+        0x0bfe04bd,
+        0x1f2af000,
+        0xb60424b6,
+        0x94bd0220,
+        0x800899f0,
+        0xf6023700,
+        0x04bd0009,
+        0x02810080,
+        0xbd0002f6,
+        0x0000d204,
+        0x25f08000,
+        0x88008002,
+        0x0002f602,
+        0x100104bd,
+        0xf0020042,
+        0x12fa0223,
+        0xbd03f805,
+        0x0899f094,
+        0x02170080,
+        0xbd0009f6,
+        0x81019804,
+        0x981814b6,
+        0x25b68002,
+        0x0512fd08,
+        0xbd1601b5,
+        0x0999f094,
+        0x02370080,
+        0xbd0009f6,
+        0x81008004,
+        0x0001f602,
+        0x010204bd,
+        0x02880080,
+        0xbd0002f6,
+        0x01004104,
+        0xfa0613f0,
+        0x03f80501,
+        0x99f094bd,
+        0x17008009,
+        0x0009f602,
+        0x94bd04bd,
+        0x800599f0,
+        0xf6021700,
+        0x04bd0009,
+/* 0x07d6: ctx_chan */
+        0xea7e00f8,
+        0x0c0a0006,
+        0x0000b87e,
+        0xd27e050f,
+        0x00f80006,
+/* 0x07e8: ctx_mmio_exec */
+        0x80410398,
+        0xf6028100,
+        0x04bd0003,
+/* 0x07f6: ctx_mmio_loop */
+        0x34c434bd,
+        0x0e1bf4ff,
+        0xf0020045,
+        0x35fa0653,
+/* 0x0807: ctx_mmio_pull */
+        0x9803f805,
+        0x4f98804e,
+        0x008f7e81,
+        0x0830b600,
+        0xf40112b6,
+/* 0x081a: ctx_mmio_done */
+        0x0398df1b,
+        0x81008016,
+        0x0003f602,
+        0x00b504bd,
+        0x01004140,
+        0xfa0613f0,
+        0x03f80601,
+/* 0x0836: ctx_xfer */
+        0x040e00f8,
+        0x03020080,
+        0xbd000ef6,
+/* 0x0841: ctx_xfer_idle */
+        0x00008e04,
+        0x00eecf03,
+        0x2000e4f1,
+        0xf4f51bf4,
+        0x02f40611,
+/* 0x0855: ctx_xfer_pre */
+        0x7e100f0c,
+        0xf40006b3,
+/* 0x085e: ctx_xfer_pre_load */
+        0x020f1b11,
+        0x0006627e,
+        0x0006717e,
+        0x0006837e,
+        0x627ef4bd,
+        0xea7e0006,
+/* 0x0876: ctx_xfer_exec */
+        0x01980006,
+        0x8024bd16,
+        0xf6010500,
+        0x04bd0002,
+        0x008e1fb2,
+        0x8f7e41a5,
+        0xfcf00000,
+        0x022cf001,
+        0xfd0124b6,
+        0xffb205f2,
+        0x41a5048e,
+        0x00008f7e,
+        0x0002167e,
+        0xfc8024bd,
+        0x02f60247,
+        0xf004bd00,
+        0x20b6012c,
+        0x4afc8003,
+        0x0002f602,
+        0xacf004bd,
+        0x06a5f001,
+        0x0c98000b,
+        0x010d9800,
+        0x3d7e000e,
+        0x080a0001,
+        0x0000ec7e,
+        0x00020a7e,
+        0x0a1201f4,
+        0x00b87e0c,
+        0x7e050f00,
+        0xf40006d2,
+/* 0x08f2: ctx_xfer_post */
+        0x020f2d02,
+        0x0006627e,
+        0xb37ef4bd,
+        0x277e0006,
+        0x717e0002,
+        0xf4bd0006,
+        0x0006627e,
+        0x981011f4,
+        0x11fd4001,
+        0x070bf405,
+        0x0007e87e,
+/* 0x091c: ctx_xfer_no_post_mmio */
+/* 0x091c: ctx_xfer_done */
+        0x000000f8,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+        0x00000000,
+};
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnv108.fuc5.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnv108.fuc5.h
index 4750984bf380..64dfd75192bf 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnv108.fuc5.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnv108.fuc5.h
@@ -342,7 +342,7 @@ uint32_t nv108_grhub_code[] = {
         0xb4f000bb,
         0x10b4b01f,
         0x0af31bf4,
-        0x00b87e02,
+        0x00b87e05,
         0x250ef400,
 /* 0x01d8: mmctx_stop */
         0xb600abc8,
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvc0.fuc.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvc0.fuc.h
index 132f684b1946..f8f7b278a13f 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvc0.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvc0.fuc.h
@@ -361,7 +361,7 @@ uint32_t nvc0_grhub_code[] = {
         0x1fb4f000,
         0xf410b4b0,
         0xa7f0f01b,
-        0xd021f402,
+        0xd021f405,
 /* 0x0223: mmctx_stop */
         0xc82b0ef4,
         0xb4b600ab,
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvd7.fuc.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvd7.fuc.h
index 84af82418987..624215a005b0 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvd7.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvd7.fuc.h
@@ -361,7 +361,7 @@ uint32_t nvd7_grhub_code[] = {
         0x1fb4f000,
         0xf410b4b0,
         0xa7f0f01b,
-        0xd021f402,
+        0xd021f405,
 /* 0x0223: mmctx_stop */
         0xc82b0ef4,
         0xb4b600ab,
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnve0.fuc.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnve0.fuc.h
index 1c179bdd48cc..6547b3dfc7ed 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnve0.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnve0.fuc.h
@@ -361,7 +361,7 @@ uint32_t nve0_grhub_code[] = {
         0x1fb4f000,
         0xf410b4b0,
         0xa7f0f01b,
-        0xd021f402,
+        0xd021f405,
 /* 0x0223: mmctx_stop */
         0xc82b0ef4,
         0xb4b600ab,
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvf0.fuc.h b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvf0.fuc.h
index 229c0ae37228..a5aee5a4302f 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvf0.fuc.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/hubnvf0.fuc.h
@@ -361,7 +361,7 @@ uint32_t nvf0_grhub_code[] = {
         0x1fb4f000,
         0xf410b4b0,
         0xa7f0f01b,
-        0xd021f402,
+        0xd021f405,
 /* 0x0223: mmctx_stop */
         0xc82b0ef4,
         0xb4b600ab,
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/macros.fuc b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/macros.fuc
index 6ffe28307dbd..a47d49db5232 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/fuc/macros.fuc
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/fuc/macros.fuc
@@ -132,6 +132,7 @@
 #define NV_PGRAPH_GPCX_GPCCS_FIFO_CMD                                  0x41a068
 #define NV_PGRAPH_GPCX_GPCCS_FIFO_ACK                                  0x41a074
 #define NV_PGRAPH_GPCX_GPCCS_UNITS                                     0x41a608
+#define NV_PGRAPH_GPCX_GPCCS_CAPS                                      0x41a108
 #define NV_PGRAPH_GPCX_GPCCS_RED_SWITCH                                0x41a614
 #define NV_PGRAPH_GPCX_GPCCS_RED_SWITCH_UNK11                        0x00000800
 #define NV_PGRAPH_GPCX_GPCCS_RED_SWITCH_ENABLE                       0x00000200
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/gm107.c b/drivers/gpu/drm/nouveau/core/engine/graph/gm107.c
new file mode 100644
index 000000000000..21c5f31d607f
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/gm107.c
@@ -0,0 +1,465 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/P0260.h>
+
+#include "nvc0.h"
+#include "ctxnvc0.h"
+
+/*******************************************************************************
+ * Graphics object classes
+ ******************************************************************************/
+
+static struct nouveau_oclass
+gm107_graph_sclass[] = {
+        { 0x902d, &nouveau_object_ofuncs },
+        { 0xa140, &nouveau_object_ofuncs },
+        { 0xb097, &nouveau_object_ofuncs },
+        { 0xb0c0, &nouveau_object_ofuncs },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH register lists
+ ******************************************************************************/
+
+static const struct nvc0_graph_init
+gm107_graph_init_main_0[] = {
+        { 0x400080,   1, 0x04, 0x003003c2 },
+        { 0x400088,   1, 0x04, 0x0001bfe7 },
+        { 0x40008c,   1, 0x04, 0x00060000 },
+        { 0x400090,   1, 0x04, 0x00000030 },
+        { 0x40013c,   1, 0x04, 0x003901f3 },
+        { 0x400140,   1, 0x04, 0x00000100 },
+        { 0x400144,   1, 0x04, 0x00000000 },
+        { 0x400148,   1, 0x04, 0x00000110 },
+        { 0x400138,   1, 0x04, 0x00000000 },
+        { 0x400130,   2, 0x04, 0x00000000 },
+        { 0x400124,   1, 0x04, 0x00000002 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_ds_0[] = {
+        { 0x405844,   1, 0x04, 0x00ffffff },
+        { 0x405850,   1, 0x04, 0x00000000 },
+        { 0x405900,   1, 0x04, 0x00000000 },
+        { 0x405908,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_scc_0[] = {
+        { 0x40803c,   1, 0x04, 0x00000010 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_sked_0[] = {
+        { 0x407010,   1, 0x04, 0x00000000 },
+        { 0x407040,   1, 0x04, 0x40440424 },
+        { 0x407048,   1, 0x04, 0x0000000a },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_prop_0[] = {
+        { 0x418408,   1, 0x04, 0x00000000 },
+        { 0x4184a0,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_setup_1[] = {
+        { 0x4188c8,   2, 0x04, 0x00000000 },
+        { 0x4188d0,   1, 0x04, 0x00010000 },
+        { 0x4188d4,   1, 0x04, 0x00010201 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_zcull_0[] = {
+        { 0x418910,   1, 0x04, 0x00010001 },
+        { 0x418914,   1, 0x04, 0x00000301 },
+        { 0x418918,   1, 0x04, 0x00800000 },
+        { 0x418930,   2, 0x04, 0x00000000 },
+        { 0x418980,   1, 0x04, 0x77777770 },
+        { 0x418984,   3, 0x04, 0x77777777 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_gpc_unk_1[] = {
+        { 0x418d00,   1, 0x04, 0x00000000 },
+        { 0x418f00,   1, 0x04, 0x00000400 },
+        { 0x418f08,   1, 0x04, 0x00000000 },
+        { 0x418e08,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_tpccs_0[] = {
+        { 0x419dc4,   1, 0x04, 0x00000000 },
+        { 0x419dc8,   1, 0x04, 0x00000501 },
+        { 0x419dd0,   1, 0x04, 0x00000000 },
+        { 0x419dd4,   1, 0x04, 0x00000100 },
+        { 0x419dd8,   1, 0x04, 0x00000001 },
+        { 0x419ddc,   1, 0x04, 0x00000002 },
+        { 0x419de0,   1, 0x04, 0x00000001 },
+        { 0x419d0c,   1, 0x04, 0x00000000 },
+        { 0x419d10,   1, 0x04, 0x00000014 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_tex_0[] = {
+        { 0x419ab0,   1, 0x04, 0x00000000 },
+        { 0x419ab8,   1, 0x04, 0x000000e7 },
+        { 0x419abc,   1, 0x04, 0x00000000 },
+        { 0x419acc,   1, 0x04, 0x000000ff },
+        { 0x419ac0,   1, 0x04, 0x00000000 },
+        { 0x419aa8,   2, 0x04, 0x00000000 },
+        { 0x419ad0,   2, 0x04, 0x00000000 },
+        { 0x419ae0,   2, 0x04, 0x00000000 },
+        { 0x419af0,   4, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_pe_0[] = {
+        { 0x419900,   1, 0x04, 0x000000ff },
+        { 0x41980c,   1, 0x04, 0x00000010 },
+        { 0x419844,   1, 0x04, 0x00000000 },
+        { 0x419838,   1, 0x04, 0x000000ff },
+        { 0x419850,   1, 0x04, 0x00000004 },
+        { 0x419854,   2, 0x04, 0x00000000 },
+        { 0x419894,   3, 0x04, 0x00100401 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_l1c_0[] = {
+        { 0x419c98,   1, 0x04, 0x00000000 },
+        { 0x419cc0,   2, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_sm_0[] = {
+        { 0x419e30,   1, 0x04, 0x000000ff },
+        { 0x419e00,   1, 0x04, 0x00000000 },
+        { 0x419ea0,   1, 0x04, 0x00000000 },
+        { 0x419ee4,   1, 0x04, 0x00000000 },
+        { 0x419ea4,   1, 0x04, 0x00000100 },
+        { 0x419ea8,   1, 0x04, 0x01000000 },
+        { 0x419ee8,   1, 0x04, 0x00000091 },
+        { 0x419eb4,   1, 0x04, 0x00000000 },
+        { 0x419ebc,   2, 0x04, 0x00000000 },
+        { 0x419edc,   1, 0x04, 0x000c1810 },
+        { 0x419ed8,   1, 0x04, 0x00000000 },
+        { 0x419ee0,   1, 0x04, 0x00000000 },
+        { 0x419f74,   1, 0x04, 0x00005155 },
+        { 0x419f80,   4, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_l1c_1[] = {
+        { 0x419ccc,   2, 0x04, 0x00000000 },
+        { 0x419c80,   1, 0x04, 0x3f006022 },
+        { 0x419c88,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_pes_0[] = {
+        { 0x41be50,   1, 0x04, 0x000000ff },
+        { 0x41be04,   1, 0x04, 0x00000000 },
+        { 0x41be08,   1, 0x04, 0x00000004 },
+        { 0x41be0c,   1, 0x04, 0x00000008 },
+        { 0x41be10,   1, 0x04, 0x0e3b8bc7 },
+        { 0x41be14,   2, 0x04, 0x00000000 },
+        { 0x41be3c,   5, 0x04, 0x00100401 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_wwdx_0[] = {
+        { 0x41bfd4,   1, 0x04, 0x00800000 },
+        { 0x41bfdc,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_cbm_0[] = {
+        { 0x41becc,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_be_0[] = {
+        { 0x408890,   1, 0x04, 0x000000ff },
+        { 0x40880c,   1, 0x04, 0x00000000 },
+        { 0x408850,   1, 0x04, 0x00000004 },
+        { 0x408878,   1, 0x04, 0x00c81603 },
+        { 0x40887c,   1, 0x04, 0x80543432 },
+        { 0x408880,   1, 0x04, 0x0010581e },
+        { 0x408884,   1, 0x04, 0x00001205 },
+        { 0x408974,   1, 0x04, 0x000000ff },
+        { 0x408910,   9, 0x04, 0x00000000 },
+        { 0x408950,   1, 0x04, 0x00000000 },
+        { 0x408954,   1, 0x04, 0x0000ffff },
+        { 0x408958,   1, 0x04, 0x00000034 },
+        { 0x40895c,   1, 0x04, 0x8531a003 },
+        { 0x408960,   1, 0x04, 0x0561985a },
+        { 0x408964,   1, 0x04, 0x04e15c4f },
+        { 0x408968,   1, 0x04, 0x02808833 },
+        { 0x40896c,   1, 0x04, 0x01f02438 },
+        { 0x408970,   1, 0x04, 0x00012c00 },
+        { 0x408984,   1, 0x04, 0x00000000 },
+        { 0x408988,   1, 0x04, 0x08040201 },
+        { 0x40898c,   1, 0x04, 0x80402010 },
+        {}
+};
+
+static const struct nvc0_graph_init
+gm107_graph_init_sm_1[] = {
+        { 0x419e5c,   1, 0x04, 0x00000000 },
+        { 0x419e58,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_pack
+gm107_graph_pack_mmio[] = {
+        { gm107_graph_init_main_0 },
+        { nvf0_graph_init_fe_0 },
+        { nvc0_graph_init_pri_0 },
+        { nvc0_graph_init_rstr2d_0 },
+        { nvc0_graph_init_pd_0 },
+        { gm107_graph_init_ds_0 },
+        { gm107_graph_init_scc_0 },
+        { gm107_graph_init_sked_0 },
+        { nvf0_graph_init_cwd_0 },
+        { gm107_graph_init_prop_0 },
+        { nv108_graph_init_gpc_unk_0 },
+        { nvc0_graph_init_setup_0 },
+        { nvc0_graph_init_crstr_0 },
+        { gm107_graph_init_setup_1 },
+        { gm107_graph_init_zcull_0 },
+        { nvc0_graph_init_gpm_0 },
+        { gm107_graph_init_gpc_unk_1 },
+        { nvc0_graph_init_gcc_0 },
+        { gm107_graph_init_tpccs_0 },
+        { gm107_graph_init_tex_0 },
+        { gm107_graph_init_pe_0 },
+        { gm107_graph_init_l1c_0 },
+        { nvc0_graph_init_mpc_0 },
+        { gm107_graph_init_sm_0 },
+        { gm107_graph_init_l1c_1 },
+        { gm107_graph_init_pes_0 },
+        { gm107_graph_init_wwdx_0 },
+        { gm107_graph_init_cbm_0 },
+        { gm107_graph_init_be_0 },
+        { gm107_graph_init_sm_1 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+static void
+gm107_graph_init_bios(struct nvc0_graph_priv *priv)
+{
+        static const struct {
+                u32 ctrl;
+                u32 data;
+        } regs[] = {
+                { 0x419ed8, 0x419ee0 },
+                { 0x419ad0, 0x419ad4 },
+                { 0x419ae0, 0x419ae4 },
+                { 0x419af0, 0x419af4 },
+                { 0x419af8, 0x419afc },
+        };
+        struct nouveau_bios *bios = nouveau_bios(priv);
+        struct nvbios_P0260E infoE;
+        struct nvbios_P0260X infoX;
+        int E = -1, X;
+        u8 ver, hdr;
+
+        while (nvbios_P0260Ep(bios, ++E, &ver, &hdr, &infoE)) {
+                if (X = -1, E < ARRAY_SIZE(regs)) {
+                        nv_wr32(priv, regs[E].ctrl, infoE.data);
+                        while (nvbios_P0260Xp(bios, ++X, &ver, &hdr, &infoX))
+                                nv_wr32(priv, regs[E].data, infoX.data);
+                }
+        }
+}
+
+int
+gm107_graph_init(struct nouveau_object *object)
+{
+        struct nvc0_graph_oclass *oclass = (void *)object->oclass;
+        struct nvc0_graph_priv *priv = (void *)object;
+        const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, priv->tpc_total);
+        u32 data[TPC_MAX / 8] = {};
+        u8  tpcnr[GPC_MAX];
+        int gpc, tpc, ppc, rop;
+        int ret, i;
+
+        ret = nouveau_graph_init(&priv->base);
+        if (ret)
+                return ret;
+
+        nv_wr32(priv, GPC_BCAST(0x0880), 0x00000000);
+        nv_wr32(priv, GPC_BCAST(0x0890), 0x00000000);
+        nv_wr32(priv, GPC_BCAST(0x0894), 0x00000000);
+        nv_wr32(priv, GPC_BCAST(0x08b4), priv->unk4188b4->addr >> 8);
+        nv_wr32(priv, GPC_BCAST(0x08b8), priv->unk4188b8->addr >> 8);
+
+        nvc0_graph_mmio(priv, oclass->mmio);
+
+        gm107_graph_init_bios(priv);
+
+        nv_wr32(priv, GPC_UNIT(0, 0x3018), 0x00000001);
+
+        memset(data, 0x00, sizeof(data));
+        memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
+        for (i = 0, gpc = -1; i < priv->tpc_total; i++) {
+                do {
+                        gpc = (gpc + 1) % priv->gpc_nr;
+                } while (!tpcnr[gpc]);
+                tpc = priv->tpc_nr[gpc] - tpcnr[gpc]--;
+
+                data[i / 8] |= tpc << ((i % 8) * 4);
+        }
+
+        nv_wr32(priv, GPC_BCAST(0x0980), data[0]);
+        nv_wr32(priv, GPC_BCAST(0x0984), data[1]);
+        nv_wr32(priv, GPC_BCAST(0x0988), data[2]);
+        nv_wr32(priv, GPC_BCAST(0x098c), data[3]);
+
+        for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+                nv_wr32(priv, GPC_UNIT(gpc, 0x0914),
+                        priv->magic_not_rop_nr << 8 | priv->tpc_nr[gpc]);
+                nv_wr32(priv, GPC_UNIT(gpc, 0x0910), 0x00040000 |
+                        priv->tpc_total);
+                nv_wr32(priv, GPC_UNIT(gpc, 0x0918), magicgpc918);
+        }
+
+        nv_wr32(priv, GPC_BCAST(0x3fd4), magicgpc918);
+        nv_wr32(priv, GPC_BCAST(0x08ac), nv_rd32(priv, 0x100800));
+
+        nv_wr32(priv, 0x400500, 0x00010001);
+
+        nv_wr32(priv, 0x400100, 0xffffffff);
+        nv_wr32(priv, 0x40013c, 0xffffffff);
+        nv_wr32(priv, 0x400124, 0x00000002);
+        nv_wr32(priv, 0x409c24, 0x000e0000);
+
+        nv_wr32(priv, 0x404000, 0xc0000000);
+        nv_wr32(priv, 0x404600, 0xc0000000);
+        nv_wr32(priv, 0x408030, 0xc0000000);
+        nv_wr32(priv, 0x404490, 0xc0000000);
+        nv_wr32(priv, 0x406018, 0xc0000000);
+        nv_wr32(priv, 0x407020, 0x40000000);
+        nv_wr32(priv, 0x405840, 0xc0000000);
+        nv_wr32(priv, 0x405844, 0x00ffffff);
+        nv_mask(priv, 0x419cc0, 0x00000008, 0x00000008);
+
+        for (gpc = 0; gpc < priv->gpc_nr; gpc++) {
+                for (ppc = 0; ppc < 2 /* priv->ppc_nr[gpc] */; ppc++)
+                        nv_wr32(priv, PPC_UNIT(gpc, ppc, 0x038), 0xc0000000);
+                nv_wr32(priv, GPC_UNIT(gpc, 0x0420), 0xc0000000);
+                nv_wr32(priv, GPC_UNIT(gpc, 0x0900), 0xc0000000);
+                nv_wr32(priv, GPC_UNIT(gpc, 0x1028), 0xc0000000);
+                nv_wr32(priv, GPC_UNIT(gpc, 0x0824), 0xc0000000);
+                for (tpc = 0; tpc < priv->tpc_nr[gpc]; tpc++) {
+                        nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
+                        nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
+                        nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
+                        nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
+                        nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
+                        nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x430), 0xc0000000);
+                        nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x644), 0x00dffffe);
+                        nv_wr32(priv, TPC_UNIT(gpc, tpc, 0x64c), 0x00000005);
+                }
+                nv_wr32(priv, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
+                nv_wr32(priv, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
+        }
+
+        for (rop = 0; rop < priv->rop_nr; rop++) {
+                nv_wr32(priv, ROP_UNIT(rop, 0x144), 0x40000000);
+                nv_wr32(priv, ROP_UNIT(rop, 0x070), 0x40000000);
+                nv_wr32(priv, ROP_UNIT(rop, 0x204), 0xffffffff);
+                nv_wr32(priv, ROP_UNIT(rop, 0x208), 0xffffffff);
+        }
+
+        nv_wr32(priv, 0x400108, 0xffffffff);
+        nv_wr32(priv, 0x400138, 0xffffffff);
+        nv_wr32(priv, 0x400118, 0xffffffff);
+        nv_wr32(priv, 0x400130, 0xffffffff);
+        nv_wr32(priv, 0x40011c, 0xffffffff);
+        nv_wr32(priv, 0x400134, 0xffffffff);
+
+        nv_wr32(priv, 0x400054, 0x2c350f63);
+        return nvc0_graph_init_ctxctl(priv);
+}
+
+#include "fuc/hubgm107.fuc5.h"
+
+static struct nvc0_graph_ucode
+gm107_graph_fecs_ucode = {
+        .code.data = gm107_grhub_code,
+        .code.size = sizeof(gm107_grhub_code),
+        .data.data = gm107_grhub_data,
+        .data.size = sizeof(gm107_grhub_data),
+};
+
+#include "fuc/gpcgm107.fuc5.h"
+
+static struct nvc0_graph_ucode
+gm107_graph_gpccs_ucode = {
+        .code.data = gm107_grgpc_code,
+        .code.size = sizeof(gm107_grgpc_code),
+        .data.data = gm107_grgpc_data,
+        .data.size = sizeof(gm107_grgpc_data),
+};
+
+struct nouveau_oclass *
+gm107_graph_oclass = &(struct nvc0_graph_oclass) {
+        .base.handle = NV_ENGINE(GR, 0x07),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nvc0_graph_ctor,
+                .dtor = nvc0_graph_dtor,
+                .init = gm107_graph_init,
+                .fini = _nouveau_graph_fini,
+        },
+        .cclass = &gm107_grctx_oclass,
+        .sclass =  gm107_graph_sclass,
+        .mmio = gm107_graph_pack_mmio,
+        .fecs.ucode = 0 ? &gm107_graph_fecs_ucode : NULL,
+        .gpccs.ucode = &gm107_graph_gpccs_ucode,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv108.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv108.c
index e1af65ead379..00ea1a089822 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv108.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv108.c
@@ -23,6 +23,7 @@
  */
 
 #include "nvc0.h"
+#include "ctxnvc0.h"
 
 /*******************************************************************************
  * Graphics object classes
@@ -38,11 +39,11 @@ nv108_graph_sclass[] = {
 };
 
 /*******************************************************************************
- * PGRAPH engine/subdev functions
+ * PGRAPH register lists
  ******************************************************************************/
 
-static struct nvc0_graph_init
-nv108_graph_init_regs[] = {
+static const struct nvc0_graph_init
+nv108_graph_init_main_0[] = {
         { 0x400080,   1, 0x04, 0x003083c2 },
         { 0x400088,   1, 0x04, 0x0001bfe7 },
         { 0x40008c,   1, 0x04, 0x00000000 },
@@ -57,66 +58,46 @@ nv108_graph_init_regs[] = {
         {}
 };
 
-struct nvc0_graph_init
-nv108_graph_init_unk58xx[] = {
+static const struct nvc0_graph_init
+nv108_graph_init_ds_0[] = {
         { 0x405844,   1, 0x04, 0x00ffffff },
         { 0x405850,   1, 0x04, 0x00000000 },
         { 0x405900,   1, 0x04, 0x00000000 },
         { 0x405908,   1, 0x04, 0x00000000 },
-        { 0x405928,   1, 0x04, 0x00000000 },
-        { 0x40592c,   1, 0x04, 0x00000000 },
+        { 0x405928,   2, 0x04, 0x00000000 },
         {}
 };
 
-static struct nvc0_graph_init
-nv108_graph_init_gpc[] = {
-        { 0x418408,   1, 0x04, 0x00000000 },
-        { 0x4184a0,   3, 0x04, 0x00000000 },
+const struct nvc0_graph_init
+nv108_graph_init_gpc_unk_0[] = {
         { 0x418604,   1, 0x04, 0x00000000 },
         { 0x418680,   1, 0x04, 0x00000000 },
         { 0x418714,   1, 0x04, 0x00000000 },
         { 0x418384,   2, 0x04, 0x00000000 },
-        { 0x418814,   3, 0x04, 0x00000000 },
-        { 0x418b04,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nv108_graph_init_setup_1[] = {
         { 0x4188c8,   2, 0x04, 0x00000000 },
         { 0x4188d0,   1, 0x04, 0x00010000 },
         { 0x4188d4,   1, 0x04, 0x00000201 },
-        { 0x418910,   1, 0x04, 0x00010001 },
-        { 0x418914,   1, 0x04, 0x00000301 },
-        { 0x418918,   1, 0x04, 0x00800000 },
-        { 0x418980,   1, 0x04, 0x77777770 },
-        { 0x418984,   3, 0x04, 0x77777777 },
-        { 0x418c04,   1, 0x04, 0x00000000 },
-        { 0x418c64,   2, 0x04, 0x00000000 },
-        { 0x418c88,   1, 0x04, 0x00000000 },
-        { 0x418cb4,   2, 0x04, 0x00000000 },
-        { 0x418d00,   1, 0x04, 0x00000000 },
-        { 0x418d28,   2, 0x04, 0x00000000 },
-        { 0x418f00,   1, 0x04, 0x00000400 },
-        { 0x418f08,   1, 0x04, 0x00000000 },
-        { 0x418f20,   2, 0x04, 0x00000000 },
-        { 0x418e00,   1, 0x04, 0x00000000 },
-        { 0x418e08,   1, 0x04, 0x00000000 },
-        { 0x418e1c,   2, 0x04, 0x00000000 },
-        { 0x41900c,   1, 0x04, 0x00000000 },
-        { 0x419018,   1, 0x04, 0x00000000 },
         {}
 };
 
-static struct nvc0_graph_init
-nv108_graph_init_tpc[] = {
-        { 0x419d0c,   1, 0x04, 0x00000000 },
-        { 0x419d10,   1, 0x04, 0x00000014 },
+static const struct nvc0_graph_init
+nv108_graph_init_tex_0[] = {
         { 0x419ab0,   1, 0x04, 0x00000000 },
         { 0x419ac8,   1, 0x04, 0x00000000 },
         { 0x419ab8,   1, 0x04, 0x000000e7 },
         { 0x419abc,   2, 0x04, 0x00000000 },
         { 0x419ab4,   1, 0x04, 0x00000000 },
         { 0x419aa8,   2, 0x04, 0x00000000 },
-        { 0x41980c,   1, 0x04, 0x00000010 },
-        { 0x419844,   1, 0x04, 0x00000000 },
-        { 0x419850,   1, 0x04, 0x00000004 },
-        { 0x419854,   2, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nv108_graph_init_l1c_0[] = {
         { 0x419c98,   1, 0x04, 0x00000000 },
         { 0x419ca8,   1, 0x04, 0x00000000 },
         { 0x419cb0,   1, 0x04, 0x01000000 },
@@ -127,22 +108,47 @@ nv108_graph_init_tpc[] = {
         { 0x419cc0,   2, 0x04, 0x00000000 },
         { 0x419c80,   1, 0x04, 0x00000230 },
         { 0x419ccc,   2, 0x04, 0x00000000 },
-        { 0x419c0c,   1, 0x04, 0x00000000 },
-        { 0x419e00,   1, 0x04, 0x00000080 },
-        { 0x419ea0,   1, 0x04, 0x00000000 },
-        { 0x419ee4,   1, 0x04, 0x00000000 },
-        { 0x419ea4,   1, 0x04, 0x00000100 },
-        { 0x419ea8,   1, 0x04, 0x00000000 },
-        { 0x419eb4,   1, 0x04, 0x00000000 },
-        { 0x419ebc,   2, 0x04, 0x00000000 },
-        { 0x419edc,   1, 0x04, 0x00000000 },
-        { 0x419f00,   1, 0x04, 0x00000000 },
-        { 0x419ed0,   1, 0x04, 0x00003234 },
-        { 0x419f74,   1, 0x04, 0x00015555 },
-        { 0x419f80,   4, 0x04, 0x00000000 },
         {}
 };
 
+static const struct nvc0_graph_pack
+nv108_graph_pack_mmio[] = {
+        { nv108_graph_init_main_0 },
+        { nvf0_graph_init_fe_0 },
+        { nvc0_graph_init_pri_0 },
+        { nvc0_graph_init_rstr2d_0 },
+        { nvd9_graph_init_pd_0 },
+        { nv108_graph_init_ds_0 },
+        { nvc0_graph_init_scc_0 },
+        { nvf0_graph_init_sked_0 },
+        { nvf0_graph_init_cwd_0 },
+        { nvd9_graph_init_prop_0 },
+        { nv108_graph_init_gpc_unk_0 },
+        { nvc0_graph_init_setup_0 },
+        { nvc0_graph_init_crstr_0 },
+        { nv108_graph_init_setup_1 },
+        { nvc0_graph_init_zcull_0 },
+        { nvd9_graph_init_gpm_0 },
+        { nvf0_graph_init_gpc_unk_1 },
+        { nvc0_graph_init_gcc_0 },
+        { nve4_graph_init_tpccs_0 },
+        { nv108_graph_init_tex_0 },
+        { nve4_graph_init_pe_0 },
+        { nv108_graph_init_l1c_0 },
+        { nvc0_graph_init_mpc_0 },
+        { nvf0_graph_init_sm_0 },
+        { nvd7_graph_init_pes_0 },
+        { nvd7_graph_init_wwdx_0 },
+        { nvd7_graph_init_cbm_0 },
+        { nve4_graph_init_be_0 },
+        { nvc0_graph_init_fe_1 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
 static int
 nv108_graph_fini(struct nouveau_object *object, bool suspend)
 {
@@ -180,25 +186,6 @@ nv108_graph_fini(struct nouveau_object *object, bool suspend)
         return nouveau_graph_fini(&priv->base, suspend);
 }
 
-static struct nvc0_graph_init *
-nv108_graph_init_mmio[] = {
-        nv108_graph_init_regs,
-        nvf0_graph_init_unk40xx,
-        nvc0_graph_init_unk44xx,
-        nvc0_graph_init_unk78xx,
-        nvc0_graph_init_unk60xx,
-        nvd9_graph_init_unk64xx,
-        nv108_graph_init_unk58xx,
-        nvc0_graph_init_unk80xx,
-        nvf0_graph_init_unk70xx,
-        nvf0_graph_init_unk5bxx,
-        nv108_graph_init_gpc,
-        nv108_graph_init_tpc,
-        nve4_graph_init_unk,
-        nve4_graph_init_unk88xx,
-        NULL
-};
-
 #include "fuc/hubnv108.fuc5.h"
 
 static struct nvc0_graph_ucode
@@ -230,7 +217,7 @@ nv108_graph_oclass = &(struct nvc0_graph_oclass) {
         },
         .cclass = &nv108_grctx_oclass,
         .sclass =  nv108_graph_sclass,
-        .mmio = nv108_graph_init_mmio,
+        .mmio = nv108_graph_pack_mmio,
         .fecs.ucode = &nv108_graph_fecs_ucode,
         .gpccs.ucode = &nv108_graph_gpccs_ucode,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv20.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv20.c
index b24559315903..d145e080899a 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv20.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv20.c
@@ -349,7 +349,7 @@ nv20_graph_init(struct nouveau_object *object)
         nv_wr32(priv, NV10_PGRAPH_SURFACE, tmp);
 
         /* begin RAM config */
-        vramsz = pci_resource_len(nv_device(priv)->pdev, 0) - 1;
+        vramsz = nv_device_resource_len(nv_device(priv), 0) - 1;
         nv_wr32(priv, 0x4009A4, nv_rd32(priv, 0x100200));
         nv_wr32(priv, 0x4009A8, nv_rd32(priv, 0x100204));
         nv_wr32(priv, NV10_PGRAPH_RDI_INDEX, 0x00EA0000);
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv40.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv40.c
index 193a5de1b482..6477fbf6a550 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv40.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv40.c
@@ -484,7 +484,7 @@ nv40_graph_init(struct nouveau_object *object)
                 engine->tile_prog(engine, i);
 
         /* begin RAM config */
-        vramsz = pci_resource_len(nv_device(priv)->pdev, 0) - 1;
+        vramsz = nv_device_resource_len(nv_device(priv), 0) - 1;
         switch (nv_device(priv)->chipset) {
         case 0x40:
                 nv_wr32(priv, 0x4009A4, nv_rd32(priv, 0x100200));
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nv50.c b/drivers/gpu/drm/nouveau/core/engine/graph/nv50.c
index 7a367c402978..2c7809e1a09b 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nv50.c
@@ -197,34 +197,35 @@ static const struct nouveau_bitfield nv50_pgraph_status[] = {
         { 0x00000080, "UNK7" },
         { 0x00000100, "CTXPROG" },
         { 0x00000200, "VFETCH" },
-        { 0x00000400, "CCACHE_UNK4" },
-        { 0x00000800, "STRMOUT_GSCHED_UNK5" },
-        { 0x00001000, "UNK14XX" },
-        { 0x00002000, "UNK24XX_CSCHED" },
-        { 0x00004000, "UNK1CXX" },
+        { 0x00000400, "CCACHE_PREGEOM" },
+        { 0x00000800, "STRMOUT_VATTR_POSTGEOM" },
+        { 0x00001000, "VCLIP" },
+        { 0x00002000, "RATTR_APLANE" },
+        { 0x00004000, "TRAST" },
         { 0x00008000, "CLIPID" },
         { 0x00010000, "ZCULL" },
         { 0x00020000, "ENG2D" },
-        { 0x00040000, "UNK34XX" },
-        { 0x00080000, "TPRAST" },
-        { 0x00100000, "TPROP" },
-        { 0x00200000, "TEX" },
-        { 0x00400000, "TPVP" },
-        { 0x00800000, "MP" },
+        { 0x00040000, "RMASK" },
+        { 0x00080000, "TPC_RAST" },
+        { 0x00100000, "TPC_PROP" },
+        { 0x00200000, "TPC_TEX" },
+        { 0x00400000, "TPC_GEOM" },
+        { 0x00800000, "TPC_MP" },
         { 0x01000000, "ROP" },
         {}
 };
 
 static const char *const nv50_pgraph_vstatus_0[] = {
-        "VFETCH", "CCACHE", "UNK4", "UNK5", "GSCHED", "STRMOUT", "UNK14XX", NULL
+        "VFETCH", "CCACHE", "PREGEOM", "POSTGEOM", "VATTR", "STRMOUT", "VCLIP",
+        NULL
 };
 
 static const char *const nv50_pgraph_vstatus_1[] = {
-        "TPRAST", "TPROP", "TEXTURE", "TPVP", "MP", NULL
+        "TPC_RAST", "TPC_PROP", "TPC_TEX", "TPC_GEOM", "TPC_MP", NULL
 };
 
 static const char *const nv50_pgraph_vstatus_2[] = {
-        "UNK24XX", "CSCHED", "UNK1CXX", "CLIPID", "ZCULL", "ENG2D", "UNK34XX",
+        "RATTR", "APLANE", "TRAST", "CLIPID", "ZCULL", "ENG2D", "RMASK",
         "ROP", NULL
 };
 
@@ -329,6 +330,15 @@ static const struct nouveau_bitfield nv50_mpc_traps[] = {
         {}
 };
 
+static const struct nouveau_bitfield nv50_tex_traps[] = {
+        { 0x00000001, "" }, /* any bit set? */
+        { 0x00000002, "FAULT" },
+        { 0x00000004, "STORAGE_TYPE_MISMATCH" },
+        { 0x00000008, "LINEAR_MISMATCH" },
+        { 0x00000020, "WRONG_MEMTYPE" },
+        {}
+};
+
 static const struct nouveau_bitfield nv50_graph_trap_m2mf[] = {
         { 0x00000001, "NOTIFY" },
         { 0x00000002, "IN" },
@@ -531,6 +541,13 @@ nv50_priv_tp_trap(struct nv50_graph_priv *priv, int type, u32 ustatus_old,
                                 for (r = ustatus_addr + 4; r <= ustatus_addr + 0x10; r += 4)
                                         nv_error(priv, "\t0x%08x: 0x%08x\n", r,
                                                 nv_rd32(priv, r));
+                                if (ustatus) {
+                                        nv_error(priv, "%s - TP%d:", name, i);
+                                        nouveau_bitfield_print(nv50_tex_traps,
+                                                               ustatus);
+                                        pr_cont("\n");
+                                        ustatus = 0;
+                                }
                         }
                         break;
                 case 7: /* MP error */
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
index a73ab209ea88..f3c7329da0a0 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.c
@@ -23,6 +23,7 @@
  */
 
 #include "nvc0.h"
+#include "ctxnvc0.h"
 
 /*******************************************************************************
  * Graphics object classes
@@ -146,11 +147,11 @@ nvc0_graph_context_dtor(struct nouveau_object *object)
 }
 
 /*******************************************************************************
- * PGRAPH engine/subdev functions
+ * PGRAPH register lists
  ******************************************************************************/
 
-struct nvc0_graph_init
-nvc0_graph_init_regs[] = {
+const struct nvc0_graph_init
+nvc0_graph_init_main_0[] = {
         { 0x400080,   1, 0x04, 0x003083c2 },
         { 0x400088,   1, 0x04, 0x00006fe7 },
         { 0x40008c,   1, 0x04, 0x00000000 },
@@ -165,95 +166,170 @@ nvc0_graph_init_regs[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc0_graph_init_unk40xx[] = {
+const struct nvc0_graph_init
+nvc0_graph_init_fe_0[] = {
         { 0x40415c,   1, 0x04, 0x00000000 },
         { 0x404170,   1, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nvc0_graph_init_unk44xx[] = {
+const struct nvc0_graph_init
+nvc0_graph_init_pri_0[] = {
         { 0x404488,   2, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nvc0_graph_init_unk78xx[] = {
+const struct nvc0_graph_init
+nvc0_graph_init_rstr2d_0[] = {
         { 0x407808,   1, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nvc0_graph_init_unk60xx[] = {
+const struct nvc0_graph_init
+nvc0_graph_init_pd_0[] = {
         { 0x406024,   1, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nvc0_graph_init_unk58xx[] = {
+const struct nvc0_graph_init
+nvc0_graph_init_ds_0[] = {
         { 0x405844,   1, 0x04, 0x00ffffff },
         { 0x405850,   1, 0x04, 0x00000000 },
         { 0x405908,   1, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nvc0_graph_init_unk80xx[] = {
+const struct nvc0_graph_init
+nvc0_graph_init_scc_0[] = {
         { 0x40803c,   1, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nvc0_graph_init_gpc[] = {
+const struct nvc0_graph_init
+nvc0_graph_init_prop_0[] = {
         { 0x4184a0,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_graph_init_gpc_unk_0[] = {
         { 0x418604,   1, 0x04, 0x00000000 },
         { 0x418680,   1, 0x04, 0x00000000 },
         { 0x418714,   1, 0x04, 0x80000000 },
         { 0x418384,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_graph_init_setup_0[] = {
         { 0x418814,   3, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_graph_init_crstr_0[] = {
         { 0x418b04,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_graph_init_setup_1[] = {
         { 0x4188c8,   1, 0x04, 0x80000000 },
         { 0x4188cc,   1, 0x04, 0x00000000 },
         { 0x4188d0,   1, 0x04, 0x00010000 },
         { 0x4188d4,   1, 0x04, 0x00000001 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_graph_init_zcull_0[] = {
         { 0x418910,   1, 0x04, 0x00010001 },
         { 0x418914,   1, 0x04, 0x00000301 },
         { 0x418918,   1, 0x04, 0x00800000 },
         { 0x418980,   1, 0x04, 0x77777770 },
         { 0x418984,   3, 0x04, 0x77777777 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_graph_init_gpm_0[] = {
         { 0x418c04,   1, 0x04, 0x00000000 },
         { 0x418c88,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_graph_init_gpc_unk_1[] = {
         { 0x418d00,   1, 0x04, 0x00000000 },
         { 0x418f08,   1, 0x04, 0x00000000 },
         { 0x418e00,   1, 0x04, 0x00000050 },
         { 0x418e08,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_graph_init_gcc_0[] = {
         { 0x41900c,   1, 0x04, 0x00000000 },
         { 0x419018,   1, 0x04, 0x00000000 },
         {}
 };
 
-static struct nvc0_graph_init
-nvc0_graph_init_tpc[] = {
+const struct nvc0_graph_init
+nvc0_graph_init_tpccs_0[] = {
         { 0x419d08,   2, 0x04, 0x00000000 },
         { 0x419d10,   1, 0x04, 0x00000014 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_graph_init_tex_0[] = {
         { 0x419ab0,   1, 0x04, 0x00000000 },
         { 0x419ab8,   1, 0x04, 0x000000e7 },
         { 0x419abc,   2, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_graph_init_pe_0[] = {
         { 0x41980c,   3, 0x04, 0x00000000 },
         { 0x419844,   1, 0x04, 0x00000000 },
         { 0x41984c,   1, 0x04, 0x00005bc5 },
         { 0x419850,   4, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_graph_init_l1c_0[] = {
         { 0x419c98,   1, 0x04, 0x00000000 },
         { 0x419ca8,   1, 0x04, 0x80000000 },
         { 0x419cb4,   1, 0x04, 0x00000000 },
         { 0x419cb8,   1, 0x04, 0x00008bf4 },
         { 0x419cbc,   1, 0x04, 0x28137606 },
         { 0x419cc0,   2, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_graph_init_wwdx_0[] = {
         { 0x419bd4,   1, 0x04, 0x00800000 },
         { 0x419bdc,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_graph_init_tpccs_1[] = {
         { 0x419d2c,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc0_graph_init_mpc_0[] = {
         { 0x419c0c,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvc0_graph_init_sm_0[] = {
         { 0x419e00,   1, 0x04, 0x00000000 },
         { 0x419ea0,   1, 0x04, 0x00000000 },
         { 0x419ea4,   1, 0x04, 0x00000100 },
@@ -270,8 +346,8 @@ nvc0_graph_init_tpc[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc0_graph_init_unk88xx[] = {
+const struct nvc0_graph_init
+nvc0_graph_init_be_0[] = {
         { 0x40880c,   1, 0x04, 0x00000000 },
         { 0x408910,   9, 0x04, 0x00000000 },
         { 0x408950,   1, 0x04, 0x00000000 },
@@ -282,18 +358,64 @@ nvc0_graph_init_unk88xx[] = {
         {}
 };
 
-struct nvc0_graph_init
-nvc0_graph_tpc_0[] = {
-        { 0x50405c,   1, 0x04, 0x00000001 },
+const struct nvc0_graph_init
+nvc0_graph_init_fe_1[] = {
+        { 0x4040f0,   1, 0x04, 0x00000000 },
         {}
 };
 
+const struct nvc0_graph_init
+nvc0_graph_init_pe_1[] = {
+        { 0x419880,   1, 0x04, 0x00000002 },
+        {}
+};
+
+static const struct nvc0_graph_pack
+nvc0_graph_pack_mmio[] = {
+        { nvc0_graph_init_main_0 },
+        { nvc0_graph_init_fe_0 },
+        { nvc0_graph_init_pri_0 },
+        { nvc0_graph_init_rstr2d_0 },
+        { nvc0_graph_init_pd_0 },
+        { nvc0_graph_init_ds_0 },
+        { nvc0_graph_init_scc_0 },
+        { nvc0_graph_init_prop_0 },
+        { nvc0_graph_init_gpc_unk_0 },
+        { nvc0_graph_init_setup_0 },
+        { nvc0_graph_init_crstr_0 },
+        { nvc0_graph_init_setup_1 },
+        { nvc0_graph_init_zcull_0 },
+        { nvc0_graph_init_gpm_0 },
+        { nvc0_graph_init_gpc_unk_1 },
+        { nvc0_graph_init_gcc_0 },
+        { nvc0_graph_init_tpccs_0 },
+        { nvc0_graph_init_tex_0 },
+        { nvc0_graph_init_pe_0 },
+        { nvc0_graph_init_l1c_0 },
+        { nvc0_graph_init_wwdx_0 },
+        { nvc0_graph_init_tpccs_1 },
+        { nvc0_graph_init_mpc_0 },
+        { nvc0_graph_init_sm_0 },
+        { nvc0_graph_init_be_0 },
+        { nvc0_graph_init_fe_1 },
+        { nvc0_graph_init_pe_1 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
 void
-nvc0_graph_mmio(struct nvc0_graph_priv *priv, struct nvc0_graph_init *init)
+nvc0_graph_mmio(struct nvc0_graph_priv *priv, const struct nvc0_graph_pack *p)
 {
-        for (; init && init->count; init++) {
-                u32 addr = init->addr, i;
-                for (i = 0; i < init->count; i++) {
+        const struct nvc0_graph_pack *pack;
+        const struct nvc0_graph_init *init;
+
+        pack_for_each_init(init, pack, p) {
+                u32 next = init->addr + init->count * init->pitch;
+                u32 addr = init->addr;
+                while (addr < next) {
                         nv_wr32(priv, addr, init->data);
                         addr += init->pitch;
                 }
@@ -301,49 +423,53 @@ nvc0_graph_mmio(struct nvc0_graph_priv *priv, struct nvc0_graph_init *init)
 }
 
 void
-nvc0_graph_icmd(struct nvc0_graph_priv *priv, struct nvc0_graph_init *init)
+nvc0_graph_icmd(struct nvc0_graph_priv *priv, const struct nvc0_graph_pack *p)
 {
-        u32 addr, data;
-        int i, j;
+        const struct nvc0_graph_pack *pack;
+        const struct nvc0_graph_init *init;
+        u32 data = 0;
 
         nv_wr32(priv, 0x400208, 0x80000000);
-        for (i = 0; init->count; init++, i++) {
-                if (!i || data != init->data) {
+
+        pack_for_each_init(init, pack, p) {
+                u32 next = init->addr + init->count * init->pitch;
+                u32 addr = init->addr;
+
+                if ((pack == p && init == p->init) || data != init->data) {
                         nv_wr32(priv, 0x400204, init->data);
                         data = init->data;
                 }
 
-                addr = init->addr;
-                for (j = 0; j < init->count; j++) {
+                while (addr < next) {
                         nv_wr32(priv, 0x400200, addr);
+                        nv_wait(priv, 0x400700, 0x00000002, 0x00000000);
                         addr += init->pitch;
-                        while (nv_rd32(priv, 0x400700) & 0x00000002) {}
                 }
         }
+
         nv_wr32(priv, 0x400208, 0x00000000);
 }
 
 void
-nvc0_graph_mthd(struct nvc0_graph_priv *priv, struct nvc0_graph_mthd *mthds)
+nvc0_graph_mthd(struct nvc0_graph_priv *priv, const struct nvc0_graph_pack *p)
 {
-        struct nvc0_graph_mthd *mthd;
-        struct nvc0_graph_init *init;
-        int i = 0, j;
-        u32 data;
-
-        while ((mthd = &mthds[i++]) && (init = mthd->init)) {
-                u32  addr = 0x80000000 | mthd->oclass;
-                for (data = 0; init->count; init++) {
-                        if (init == mthd->init || data != init->data) {
-                                nv_wr32(priv, 0x40448c, init->data);
-                                data = init->data;
-                        }
+        const struct nvc0_graph_pack *pack;
+        const struct nvc0_graph_init *init;
+        u32 data = 0;
 
-                        addr = (addr & 0x8000ffff) | (init->addr << 14);
-                        for (j = 0; j < init->count; j++) {
-                                nv_wr32(priv, 0x404488, addr);
-                                addr += init->pitch << 14;
-                        }
+        pack_for_each_init(init, pack, p) {
+                u32 ctrl = 0x80000000 | pack->type;
+                u32 next = init->addr + init->count * init->pitch;
+                u32 addr = init->addr;
+
+                if ((pack == p && init == p->init) || data != init->data) {
+                        nv_wr32(priv, 0x40448c, init->data);
+                        data = init->data;
+                }
+
+                while (addr < next) {
+                        nv_wr32(priv, 0x404488, ctrl | (addr << 14));
+                        addr += init->pitch;
                 }
         }
 }
@@ -772,11 +898,12 @@ nvc0_graph_init_fw(struct nvc0_graph_priv *priv, u32 fuc_base,
 
 static void
 nvc0_graph_init_csdata(struct nvc0_graph_priv *priv,
-                       struct nvc0_graph_init *init,
+                       const struct nvc0_graph_pack *pack,
                        u32 falcon, u32 starstar, u32 base)
 {
-        u32 addr = init->addr;
-        u32 next = addr;
+        const struct nvc0_graph_pack *iter;
+        const struct nvc0_graph_init *init;
+        u32 addr = ~0, prev = ~0, xfer = 0;
         u32 star, temp;
 
         nv_wr32(priv, falcon + 0x01c0, 0x02000000 + starstar);
@@ -786,22 +913,28 @@ nvc0_graph_init_csdata(struct nvc0_graph_priv *priv,
                 star = temp;
         nv_wr32(priv, falcon + 0x01c0, 0x01000000 + star);
 
-        do {
-                if (init->addr != next) {
-                        while (addr < next) {
-                                u32 nr = min((int)(next - addr) / 4, 32);
-                                nv_wr32(priv, falcon + 0x01c4,
-                                        ((nr - 1) << 26) | (addr - base));
-                                addr += nr * 4;
-                                star += 4;
+        pack_for_each_init(init, iter, pack) {
+                u32 head = init->addr - base;
+                u32 tail = head + init->count * init->pitch;
+                while (head < tail) {
+                        if (head != prev + 4 || xfer >= 32) {
+                                if (xfer) {
+                                        u32 data = ((--xfer << 26) | addr);
+                                        nv_wr32(priv, falcon + 0x01c4, data);
+                                        star += 4;
+                                }
+                                addr = head;
+                                xfer = 0;
                         }
-                        addr = next = init->addr;
+                        prev = head;
+                        xfer = xfer + 1;
+                        head = head + init->pitch;
                 }
-                next += init->count * 4;
-        } while ((init++)->count);
+        }
 
+        nv_wr32(priv, falcon + 0x01c4, (--xfer << 26) | addr);
         nv_wr32(priv, falcon + 0x01c0, 0x01000004 + starstar);
-        nv_wr32(priv, falcon + 0x01c4, star);
+        nv_wr32(priv, falcon + 0x01c4, star + 4);
 }
 
 int
@@ -809,7 +942,6 @@ nvc0_graph_init_ctxctl(struct nvc0_graph_priv *priv)
 {
         struct nvc0_graph_oclass *oclass = (void *)nv_object(priv)->oclass;
         struct nvc0_grctx_oclass *cclass = (void *)nv_engine(priv)->cclass;
-        struct nvc0_graph_init *init;
         u32 r000260;
         int i;
 
@@ -919,10 +1051,6 @@ nvc0_graph_init_ctxctl(struct nvc0_graph_priv *priv)
                 nv_wr32(priv, 0x409184, oclass->fecs.ucode->code.data[i]);
         }
 
-        for (i = 0; (init = cclass->hub[i]); i++) {
-                nvc0_graph_init_csdata(priv, init, 0x409000, 0x000, 0x000000);
-        }
-
         /* load GPC microcode */
         nv_wr32(priv, 0x41a1c0, 0x01000000);
         for (i = 0; i < oclass->gpccs.ucode->data.size / 4; i++)
@@ -936,12 +1064,11 @@ nvc0_graph_init_ctxctl(struct nvc0_graph_priv *priv)
         }
         nv_wr32(priv, 0x000260, r000260);
 
-        if ((init = cclass->gpc[0]))
-                nvc0_graph_init_csdata(priv, init, 0x41a000, 0x000, 0x418000);
-        if ((init = cclass->gpc[2]))
-                nvc0_graph_init_csdata(priv, init, 0x41a000, 0x004, 0x419800);
-        if ((init = cclass->gpc[3]))
-                nvc0_graph_init_csdata(priv, init, 0x41a000, 0x008, 0x41be00);
+        /* load register lists */
+        nvc0_graph_init_csdata(priv, cclass->hub, 0x409000, 0x000, 0x000000);
+        nvc0_graph_init_csdata(priv, cclass->gpc, 0x41a000, 0x000, 0x418000);
+        nvc0_graph_init_csdata(priv, cclass->tpc, 0x41a000, 0x004, 0x419800);
+        nvc0_graph_init_csdata(priv, cclass->ppc, 0x41a000, 0x008, 0x41be00);
 
         /* start HUB ucode running, it'll init the GPCs */
         nv_wr32(priv, 0x40910c, 0x00000000);
@@ -988,8 +1115,7 @@ nvc0_graph_init(struct nouveau_object *object)
         nv_wr32(priv, GPC_BCAST(0x08b4), priv->unk4188b4->addr >> 8);
         nv_wr32(priv, GPC_BCAST(0x08b8), priv->unk4188b8->addr >> 8);
 
-        for (i = 0; oclass->mmio[i]; i++)
-                nvc0_graph_mmio(priv, oclass->mmio[i]);
+        nvc0_graph_mmio(priv, oclass->mmio);
 
         memcpy(tpcnr, priv->tpc_nr, sizeof(priv->tpc_nr));
         for (i = 0, gpc = -1; i < priv->tpc_total; i++) {
@@ -1091,10 +1217,10 @@ nvc0_graph_ctor_fw(struct nvc0_graph_priv *priv, const char *fwname,
         int ret;
 
         snprintf(f, sizeof(f), "nouveau/nv%02x_%s", device->chipset, fwname);
-        ret = request_firmware(&fw, f, &device->pdev->dev);
+        ret = request_firmware(&fw, f, nv_device_base(device));
         if (ret) {
                 snprintf(f, sizeof(f), "nouveau/%s", fwname);
-                ret = request_firmware(&fw, f, &device->pdev->dev);
+                ret = request_firmware(&fw, f, nv_device_base(device));
                 if (ret) {
                         nv_error(priv, "failed to load %s\n", fwname);
                         return ret;
@@ -1220,22 +1346,6 @@ nvc0_graph_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         return 0;
 }
 
-struct nvc0_graph_init *
-nvc0_graph_init_mmio[] = {
-        nvc0_graph_init_regs,
-        nvc0_graph_init_unk40xx,
-        nvc0_graph_init_unk44xx,
-        nvc0_graph_init_unk78xx,
-        nvc0_graph_init_unk60xx,
-        nvc0_graph_init_unk58xx,
-        nvc0_graph_init_unk80xx,
-        nvc0_graph_init_gpc,
-        nvc0_graph_init_tpc,
-        nvc0_graph_init_unk88xx,
-        nvc0_graph_tpc_0,
-        NULL
-};
-
 #include "fuc/hubnvc0.fuc.h"
 
 struct nvc0_graph_ucode
@@ -1267,7 +1377,7 @@ nvc0_graph_oclass = &(struct nvc0_graph_oclass) {
         },
         .cclass = &nvc0_grctx_oclass,
         .sclass =  nvc0_graph_sclass,
-        .mmio = nvc0_graph_init_mmio,
+        .mmio = nvc0_graph_pack_mmio,
         .fecs.ucode = &nvc0_graph_fecs_ucode,
         .gpccs.ucode = &nvc0_graph_gpccs_ucode,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.h b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.h
index b0ab6de270b2..90d44616c876 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.h
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvc0.h
@@ -45,6 +45,7 @@
 #define ROP_UNIT(u, r)    (0x410000 + (u) * 0x400 + (r))
 #define GPC_BCAST(r)      (0x418000 + (r))
 #define GPC_UNIT(t, r)    (0x500000 + (t) * 0x8000 + (r))
+#define PPC_UNIT(t, m, r) (0x503000 + (t) * 0x8000 + (m) * 0x200 + (r))
 #define TPC_UNIT(t, m, r) (0x504000 + (t) * 0x8000 + (m) * 0x800 + (r))
 
 struct nvc0_graph_data {
@@ -102,8 +103,6 @@ struct nvc0_graph_chan {
         } data[4];
 };
 
-int  nvc0_grctx_generate(struct nvc0_graph_priv *);
-
 int  nvc0_graph_context_ctor(struct nouveau_object *, struct nouveau_object *,
                              struct nouveau_oclass *, void *, u32,
                              struct nouveau_object **);
@@ -130,34 +129,14 @@ struct nvc0_graph_init {
         u32 data;
 };
 
-struct nvc0_graph_mthd {
-        u16 oclass;
-        struct nvc0_graph_init *init;
-};
-
-struct nvc0_grctx {
-        struct nvc0_graph_priv *priv;
-        struct nvc0_graph_data *data;
-        struct nvc0_graph_mmio *mmio;
-        int buffer_nr;
-        u64 buffer[4];
-        u64 addr;
+struct nvc0_graph_pack {
+        const struct nvc0_graph_init *init;
+        u32 type;
 };
 
-struct nvc0_grctx_oclass {
-        struct nouveau_oclass base;
-        /* main context generation function */
-        void  (*main)(struct nvc0_graph_priv *, struct nvc0_grctx *);
-        /* context-specific modify-on-first-load list generation function */
-        void  (*mods)(struct nvc0_graph_priv *, struct nvc0_grctx *);
-        void  (*unkn)(struct nvc0_graph_priv *);
-        /* mmio context data */
-        struct nvc0_graph_init **hub;
-        struct nvc0_graph_init **gpc;
-        /* indirect context data, generated with icmds/mthds */
-        struct nvc0_graph_init *icmd;
-        struct nvc0_graph_mthd *mthd;
-};
+#define pack_for_each_init(init, pack, head)                                   \
+        for (pack = head; pack && pack->init; pack++)                          \
+                  for (init = pack->init; init && init->count; init++)
 
 struct nvc0_graph_ucode {
         struct nvc0_graph_fuc code;
@@ -171,7 +150,7 @@ struct nvc0_graph_oclass {
         struct nouveau_oclass base;
         struct nouveau_oclass **cclass;
         struct nouveau_oclass *sclass;
-        struct nvc0_graph_init **mmio;
+        const struct nvc0_graph_pack *mmio;
         struct {
                 struct nvc0_graph_ucode *ucode;
         } fecs;
@@ -180,119 +159,72 @@ struct nvc0_graph_oclass {
         } gpccs;
 };
 
-void nvc0_graph_mmio(struct nvc0_graph_priv *, struct nvc0_graph_init *);
-void nvc0_graph_icmd(struct nvc0_graph_priv *, struct nvc0_graph_init *);
-void nvc0_graph_mthd(struct nvc0_graph_priv *, struct nvc0_graph_mthd *);
+void nvc0_graph_mmio(struct nvc0_graph_priv *, const struct nvc0_graph_pack *);
+void nvc0_graph_icmd(struct nvc0_graph_priv *, const struct nvc0_graph_pack *);
+void nvc0_graph_mthd(struct nvc0_graph_priv *, const struct nvc0_graph_pack *);
 int  nvc0_graph_init_ctxctl(struct nvc0_graph_priv *);
 
-extern struct nvc0_graph_init nvc0_graph_init_regs[];
-extern struct nvc0_graph_init nvc0_graph_init_unk40xx[];
-extern struct nvc0_graph_init nvc0_graph_init_unk44xx[];
-extern struct nvc0_graph_init nvc0_graph_init_unk78xx[];
-extern struct nvc0_graph_init nvc0_graph_init_unk60xx[];
-extern struct nvc0_graph_init nvc0_graph_init_unk58xx[];
-extern struct nvc0_graph_init nvc0_graph_init_unk80xx[];
-extern struct nvc0_graph_init nvc0_graph_init_gpc[];
-extern struct nvc0_graph_init nvc0_graph_init_unk88xx[];
-extern struct nvc0_graph_init nvc0_graph_tpc_0[];
-
-extern struct nvc0_graph_init nvc3_graph_init_unk58xx[];
-
-extern struct nvc0_graph_init nvd9_graph_init_unk58xx[];
-extern struct nvc0_graph_init nvd9_graph_init_unk64xx[];
-
-extern struct nvc0_graph_init nve4_graph_init_regs[];
-extern struct nvc0_graph_init nve4_graph_init_unk[];
-extern struct nvc0_graph_init nve4_graph_init_unk88xx[];
-
-extern struct nvc0_graph_init nvf0_graph_init_unk40xx[];
-extern struct nvc0_graph_init nvf0_graph_init_unk70xx[];
-extern struct nvc0_graph_init nvf0_graph_init_unk5bxx[];
-extern struct nvc0_graph_init nvf0_graph_init_tpc[];
-
-int  nvc0_grctx_generate(struct nvc0_graph_priv *);
-void nvc0_grctx_generate_main(struct nvc0_graph_priv *, struct nvc0_grctx *);
-void nvc0_grctx_generate_mods(struct nvc0_graph_priv *, struct nvc0_grctx *);
-void nvc0_grctx_generate_unkn(struct nvc0_graph_priv *);
-void nvc0_grctx_generate_tpcid(struct nvc0_graph_priv *);
-void nvc0_grctx_generate_r406028(struct nvc0_graph_priv *);
-void nvc0_grctx_generate_r4060a8(struct nvc0_graph_priv *);
-void nvc0_grctx_generate_r418bb8(struct nvc0_graph_priv *);
-void nve4_grctx_generate_r418bb8(struct nvc0_graph_priv *);
-void nvc0_grctx_generate_r406800(struct nvc0_graph_priv *);
-
-extern struct nouveau_oclass *nvc0_grctx_oclass;
-extern struct nvc0_graph_init *nvc0_grctx_init_hub[];
-extern struct nvc0_graph_init nvc0_grctx_init_base[];
-extern struct nvc0_graph_init nvc0_grctx_init_unk40xx[];
-extern struct nvc0_graph_init nvc0_grctx_init_unk44xx[];
-extern struct nvc0_graph_init nvc0_grctx_init_unk46xx[];
-extern struct nvc0_graph_init nvc0_grctx_init_unk47xx[];
-extern struct nvc0_graph_init nvc0_grctx_init_unk60xx[];
-extern struct nvc0_graph_init nvc0_grctx_init_unk64xx[];
-extern struct nvc0_graph_init nvc0_grctx_init_unk78xx[];
-extern struct nvc0_graph_init nvc0_grctx_init_unk80xx[];
-extern struct nvc0_graph_init nvc0_grctx_init_gpc_0[];
-extern struct nvc0_graph_init nvc0_grctx_init_gpc_1[];
-extern struct nvc0_graph_init nvc0_grctx_init_tpc[];
-extern struct nvc0_graph_init nvc0_grctx_init_icmd[];
-extern struct nvc0_graph_init nvd9_grctx_init_icmd[]; //
-
-extern struct nvc0_graph_mthd nvc0_grctx_init_mthd[];
-extern struct nvc0_graph_init nvc0_grctx_init_902d[];
-extern struct nvc0_graph_init nvc0_grctx_init_9039[];
-extern struct nvc0_graph_init nvc0_grctx_init_90c0[];
-extern struct nvc0_graph_init nvc0_grctx_init_mthd_magic[];
-
-void nvc1_grctx_generate_mods(struct nvc0_graph_priv *, struct nvc0_grctx *);
-void nvc1_grctx_generate_unkn(struct nvc0_graph_priv *);
-extern struct nouveau_oclass *nvc1_grctx_oclass;
-extern struct nvc0_graph_init nvc1_grctx_init_9097[];
-
-extern struct nouveau_oclass *nvc3_grctx_oclass;
-
-extern struct nouveau_oclass *nvc8_grctx_oclass;
-extern struct nvc0_graph_init nvc8_grctx_init_9197[];
-extern struct nvc0_graph_init nvc8_grctx_init_9297[];
-
-extern struct nouveau_oclass *nvd7_grctx_oclass;
-
-extern struct nouveau_oclass *nvd9_grctx_oclass;
-extern struct nvc0_graph_init nvd9_grctx_init_rop[];
-extern struct nvc0_graph_mthd nvd9_grctx_init_mthd[];
-
-void nve4_grctx_generate_main(struct nvc0_graph_priv *, struct nvc0_grctx *);
-void nve4_grctx_generate_unkn(struct nvc0_graph_priv *);
-extern struct nouveau_oclass *nve4_grctx_oclass;
-extern struct nvc0_graph_init nve4_grctx_init_unk46xx[];
-extern struct nvc0_graph_init nve4_grctx_init_unk47xx[];
-extern struct nvc0_graph_init nve4_grctx_init_unk58xx[];
-extern struct nvc0_graph_init nve4_grctx_init_unk80xx[];
-extern struct nvc0_graph_init nve4_grctx_init_unk90xx[];
-
-extern struct nouveau_oclass *nvf0_grctx_oclass;
-extern struct nvc0_graph_init nvf0_grctx_init_unk44xx[];
-extern struct nvc0_graph_init nvf0_grctx_init_unk5bxx[];
-extern struct nvc0_graph_init nvf0_grctx_init_unk60xx[];
-
-extern struct nouveau_oclass *nv108_grctx_oclass;
-
-#define mmio_data(s,a,p) do {                                                  \
-        info->buffer[info->buffer_nr] = round_up(info->addr, (a));             \
-        info->addr = info->buffer[info->buffer_nr++] + (s);                    \
-        info->data->size = (s);                                                \
-        info->data->align = (a);                                               \
-        info->data->access = (p);                                              \
-        info->data++;                                                          \
-} while(0)
-
-#define mmio_list(r,d,s,b) do {                                                \
-        info->mmio->addr = (r);                                                \
-        info->mmio->data = (d);                                                \
-        info->mmio->shift = (s);                                               \
-        info->mmio->buffer = (b);                                              \
-        info->mmio++;                                                          \
-        nv_wr32(priv, (r), (d) | ((s) ? (info->buffer[(b)] >> (s)) : 0));      \
-} while(0)
+/* register init value lists */
+
+extern const struct nvc0_graph_init nvc0_graph_init_main_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_fe_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_pri_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_rstr2d_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_pd_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_ds_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_scc_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_prop_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_gpc_unk_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_setup_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_crstr_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_setup_1[];
+extern const struct nvc0_graph_init nvc0_graph_init_zcull_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_gpm_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_gpc_unk_1[];
+extern const struct nvc0_graph_init nvc0_graph_init_gcc_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_tpccs_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_tex_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_pe_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_l1c_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_wwdx_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_tpccs_1[];
+extern const struct nvc0_graph_init nvc0_graph_init_mpc_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_be_0[];
+extern const struct nvc0_graph_init nvc0_graph_init_fe_1[];
+extern const struct nvc0_graph_init nvc0_graph_init_pe_1[];
+
+extern const struct nvc0_graph_init nvc4_graph_init_ds_0[];
+extern const struct nvc0_graph_init nvc4_graph_init_tex_0[];
+extern const struct nvc0_graph_init nvc4_graph_init_sm_0[];
+
+extern const struct nvc0_graph_init nvc1_graph_init_gpc_unk_0[];
+extern const struct nvc0_graph_init nvc1_graph_init_setup_1[];
+
+extern const struct nvc0_graph_init nvd9_graph_init_pd_0[];
+extern const struct nvc0_graph_init nvd9_graph_init_ds_0[];
+extern const struct nvc0_graph_init nvd9_graph_init_prop_0[];
+extern const struct nvc0_graph_init nvd9_graph_init_gpm_0[];
+extern const struct nvc0_graph_init nvd9_graph_init_gpc_unk_1[];
+extern const struct nvc0_graph_init nvd9_graph_init_tex_0[];
+extern const struct nvc0_graph_init nvd9_graph_init_sm_0[];
+extern const struct nvc0_graph_init nvd9_graph_init_fe_1[];
+
+extern const struct nvc0_graph_init nvd7_graph_init_pes_0[];
+extern const struct nvc0_graph_init nvd7_graph_init_wwdx_0[];
+extern const struct nvc0_graph_init nvd7_graph_init_cbm_0[];
+
+extern const struct nvc0_graph_init nve4_graph_init_main_0[];
+extern const struct nvc0_graph_init nve4_graph_init_tpccs_0[];
+extern const struct nvc0_graph_init nve4_graph_init_pe_0[];
+extern const struct nvc0_graph_init nve4_graph_init_be_0[];
+
+extern const struct nvc0_graph_init nvf0_graph_init_fe_0[];
+extern const struct nvc0_graph_init nvf0_graph_init_sked_0[];
+extern const struct nvc0_graph_init nvf0_graph_init_cwd_0[];
+extern const struct nvc0_graph_init nvf0_graph_init_gpc_unk_1[];
+extern const struct nvc0_graph_init nvf0_graph_init_sm_0[];
+
+extern const struct nvc0_graph_init nv108_graph_init_gpc_unk_0[];
+
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvc1.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvc1.c
index bc4a469b86cb..30cab0b2eba1 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvc1.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvc1.c
@@ -23,6 +23,7 @@
  */
 
 #include "nvc0.h"
+#include "ctxnvc0.h"
 
 /*******************************************************************************
  * Graphics object classes
@@ -39,94 +40,82 @@ nvc1_graph_sclass[] = {
 };
 
 /*******************************************************************************
- * PGRAPH engine/subdev functions
+ * PGRAPH register lists
  ******************************************************************************/
 
-static struct nvc0_graph_init
-nvc1_graph_init_gpc[] = {
-        { 0x4184a0,   1, 0x04, 0x00000000 },
+const struct nvc0_graph_init
+nvc1_graph_init_gpc_unk_0[] = {
         { 0x418604,   1, 0x04, 0x00000000 },
         { 0x418680,   1, 0x04, 0x00000000 },
         { 0x418714,   1, 0x04, 0x00000000 },
         { 0x418384,   1, 0x04, 0x00000000 },
-        { 0x418814,   3, 0x04, 0x00000000 },
-        { 0x418b04,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc1_graph_init_setup_1[] = {
         { 0x4188c8,   2, 0x04, 0x00000000 },
         { 0x4188d0,   1, 0x04, 0x00010000 },
         { 0x4188d4,   1, 0x04, 0x00000001 },
-        { 0x418910,   1, 0x04, 0x00010001 },
-        { 0x418914,   1, 0x04, 0x00000301 },
-        { 0x418918,   1, 0x04, 0x00800000 },
-        { 0x418980,   1, 0x04, 0x77777770 },
-        { 0x418984,   3, 0x04, 0x77777777 },
-        { 0x418c04,   1, 0x04, 0x00000000 },
-        { 0x418c88,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvc1_graph_init_gpc_unk_1[] = {
         { 0x418d00,   1, 0x04, 0x00000000 },
         { 0x418f08,   1, 0x04, 0x00000000 },
         { 0x418e00,   1, 0x04, 0x00000003 },
         { 0x418e08,   1, 0x04, 0x00000000 },
-        { 0x41900c,   1, 0x04, 0x00000000 },
-        { 0x419018,   1, 0x04, 0x00000000 },
         {}
 };
 
-static struct nvc0_graph_init
-nvc1_graph_init_tpc[] = {
-        { 0x419d08,   2, 0x04, 0x00000000 },
-        { 0x419d10,   1, 0x04, 0x00000014 },
-        { 0x419ab0,   1, 0x04, 0x00000000 },
-        { 0x419ac8,   1, 0x04, 0x00000000 },
-        { 0x419ab8,   1, 0x04, 0x000000e7 },
-        { 0x419abc,   2, 0x04, 0x00000000 },
-        { 0x41980c,   2, 0x04, 0x00000000 },
+static const struct nvc0_graph_init
+nvc1_graph_init_pe_0[] = {
+        { 0x41980c,   1, 0x04, 0x00000010 },
+        { 0x419810,   1, 0x04, 0x00000000 },
         { 0x419814,   1, 0x04, 0x00000004 },
         { 0x419844,   1, 0x04, 0x00000000 },
         { 0x41984c,   1, 0x04, 0x00005bc5 },
         { 0x419850,   4, 0x04, 0x00000000 },
         { 0x419880,   1, 0x04, 0x00000002 },
-        { 0x419c98,   1, 0x04, 0x00000000 },
-        { 0x419ca8,   1, 0x04, 0x80000000 },
-        { 0x419cb4,   1, 0x04, 0x00000000 },
-        { 0x419cb8,   1, 0x04, 0x00008bf4 },
-        { 0x419cbc,   1, 0x04, 0x28137606 },
-        { 0x419cc0,   2, 0x04, 0x00000000 },
-        { 0x419bd4,   1, 0x04, 0x00800000 },
-        { 0x419bdc,   1, 0x04, 0x00000000 },
-        { 0x419d2c,   1, 0x04, 0x00000000 },
-        { 0x419c0c,   1, 0x04, 0x00000000 },
-        { 0x419e00,   1, 0x04, 0x00000000 },
-        { 0x419ea0,   1, 0x04, 0x00000000 },
-        { 0x419ea4,   1, 0x04, 0x00000100 },
-        { 0x419ea8,   1, 0x04, 0x00001100 },
-        { 0x419eac,   1, 0x04, 0x11100702 },
-        { 0x419eb0,   1, 0x04, 0x00000003 },
-        { 0x419eb4,   4, 0x04, 0x00000000 },
-        { 0x419ec8,   1, 0x04, 0x0e063818 },
-        { 0x419ecc,   1, 0x04, 0x0e060e06 },
-        { 0x419ed0,   1, 0x04, 0x00003818 },
-        { 0x419ed4,   1, 0x04, 0x011104f1 },
-        { 0x419edc,   1, 0x04, 0x00000000 },
-        { 0x419f00,   1, 0x04, 0x00000000 },
-        { 0x419f2c,   1, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init *
-nvc1_graph_init_mmio[] = {
-        nvc0_graph_init_regs,
-        nvc0_graph_init_unk40xx,
-        nvc0_graph_init_unk44xx,
-        nvc0_graph_init_unk78xx,
-        nvc0_graph_init_unk60xx,
-        nvc3_graph_init_unk58xx,
-        nvc0_graph_init_unk80xx,
-        nvc1_graph_init_gpc,
-        nvc1_graph_init_tpc,
-        nvc0_graph_init_unk88xx,
-        nvc0_graph_tpc_0,
-        NULL
+static const struct nvc0_graph_pack
+nvc1_graph_pack_mmio[] = {
+        { nvc0_graph_init_main_0 },
+        { nvc0_graph_init_fe_0 },
+        { nvc0_graph_init_pri_0 },
+        { nvc0_graph_init_rstr2d_0 },
+        { nvc0_graph_init_pd_0 },
+        { nvc4_graph_init_ds_0 },
+        { nvc0_graph_init_scc_0 },
+        { nvc0_graph_init_prop_0 },
+        { nvc1_graph_init_gpc_unk_0 },
+        { nvc0_graph_init_setup_0 },
+        { nvc0_graph_init_crstr_0 },
+        { nvc1_graph_init_setup_1 },
+        { nvc0_graph_init_zcull_0 },
+        { nvc0_graph_init_gpm_0 },
+        { nvc1_graph_init_gpc_unk_1 },
+        { nvc0_graph_init_gcc_0 },
+        { nvc0_graph_init_tpccs_0 },
+        { nvc4_graph_init_tex_0 },
+        { nvc1_graph_init_pe_0 },
+        { nvc0_graph_init_l1c_0 },
+        { nvc0_graph_init_wwdx_0 },
+        { nvc0_graph_init_tpccs_1 },
+        { nvc0_graph_init_mpc_0 },
+        { nvc4_graph_init_sm_0 },
+        { nvc0_graph_init_be_0 },
+        { nvc0_graph_init_fe_1 },
+        {}
 };
 
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
 struct nouveau_oclass *
 nvc1_graph_oclass = &(struct nvc0_graph_oclass) {
         .base.handle = NV_ENGINE(GR, 0xc1),
@@ -138,7 +127,7 @@ nvc1_graph_oclass = &(struct nvc0_graph_oclass) {
         },
         .cclass = &nvc1_grctx_oclass,
         .sclass = nvc1_graph_sclass,
-        .mmio = nvc1_graph_init_mmio,
+        .mmio = nvc1_graph_pack_mmio,
         .fecs.ucode = &nvc0_graph_fecs_ucode,
         .gpccs.ucode = &nvc0_graph_gpccs_ucode,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvc3.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvc4.c
index d44b3b3ee800..e82e70c53132 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvc3.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvc4.c
@@ -23,13 +23,14 @@
  */
 
 #include "nvc0.h"
+#include "ctxnvc0.h"
 
 /*******************************************************************************
- * PGRAPH engine/subdev functions
+ * PGRAPH register lists
  ******************************************************************************/
 
-struct nvc0_graph_init
-nvc3_graph_init_unk58xx[] = {
+const struct nvc0_graph_init
+nvc4_graph_init_ds_0[] = {
         { 0x405844,   1, 0x04, 0x00ffffff },
         { 0x405850,   1, 0x04, 0x00000000 },
         { 0x405900,   1, 0x04, 0x00002834 },
@@ -37,29 +38,27 @@ nvc3_graph_init_unk58xx[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nvc3_graph_init_tpc[] = {
-        { 0x419d08,   2, 0x04, 0x00000000 },
-        { 0x419d10,   1, 0x04, 0x00000014 },
+const struct nvc0_graph_init
+nvc4_graph_init_tex_0[] = {
         { 0x419ab0,   1, 0x04, 0x00000000 },
         { 0x419ac8,   1, 0x04, 0x00000000 },
         { 0x419ab8,   1, 0x04, 0x000000e7 },
         { 0x419abc,   2, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvc4_graph_init_pe_0[] = {
         { 0x41980c,   3, 0x04, 0x00000000 },
         { 0x419844,   1, 0x04, 0x00000000 },
         { 0x41984c,   1, 0x04, 0x00005bc5 },
         { 0x419850,   4, 0x04, 0x00000000 },
         { 0x419880,   1, 0x04, 0x00000002 },
-        { 0x419c98,   1, 0x04, 0x00000000 },
-        { 0x419ca8,   1, 0x04, 0x80000000 },
-        { 0x419cb4,   1, 0x04, 0x00000000 },
-        { 0x419cb8,   1, 0x04, 0x00008bf4 },
-        { 0x419cbc,   1, 0x04, 0x28137606 },
-        { 0x419cc0,   2, 0x04, 0x00000000 },
-        { 0x419bd4,   1, 0x04, 0x00800000 },
-        { 0x419bdc,   1, 0x04, 0x00000000 },
-        { 0x419d2c,   1, 0x04, 0x00000000 },
-        { 0x419c0c,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvc4_graph_init_sm_0[] = {
         { 0x419e00,   1, 0x04, 0x00000000 },
         { 0x419ea0,   1, 0x04, 0x00000000 },
         { 0x419ea4,   1, 0x04, 0x00000100 },
@@ -77,24 +76,43 @@ nvc3_graph_init_tpc[] = {
         {}
 };
 
-static struct nvc0_graph_init *
-nvc3_graph_init_mmio[] = {
-        nvc0_graph_init_regs,
-        nvc0_graph_init_unk40xx,
-        nvc0_graph_init_unk44xx,
-        nvc0_graph_init_unk78xx,
-        nvc0_graph_init_unk60xx,
-        nvc3_graph_init_unk58xx,
-        nvc0_graph_init_unk80xx,
-        nvc0_graph_init_gpc,
-        nvc3_graph_init_tpc,
-        nvc0_graph_init_unk88xx,
-        nvc0_graph_tpc_0,
-        NULL
+static const struct nvc0_graph_pack
+nvc4_graph_pack_mmio[] = {
+        { nvc0_graph_init_main_0 },
+        { nvc0_graph_init_fe_0 },
+        { nvc0_graph_init_pri_0 },
+        { nvc0_graph_init_rstr2d_0 },
+        { nvc0_graph_init_pd_0 },
+        { nvc4_graph_init_ds_0 },
+        { nvc0_graph_init_scc_0 },
+        { nvc0_graph_init_prop_0 },
+        { nvc0_graph_init_gpc_unk_0 },
+        { nvc0_graph_init_setup_0 },
+        { nvc0_graph_init_crstr_0 },
+        { nvc0_graph_init_setup_1 },
+        { nvc0_graph_init_zcull_0 },
+        { nvc0_graph_init_gpm_0 },
+        { nvc0_graph_init_gpc_unk_1 },
+        { nvc0_graph_init_gcc_0 },
+        { nvc0_graph_init_tpccs_0 },
+        { nvc4_graph_init_tex_0 },
+        { nvc4_graph_init_pe_0 },
+        { nvc0_graph_init_l1c_0 },
+        { nvc0_graph_init_wwdx_0 },
+        { nvc0_graph_init_tpccs_1 },
+        { nvc0_graph_init_mpc_0 },
+        { nvc4_graph_init_sm_0 },
+        { nvc0_graph_init_be_0 },
+        { nvc0_graph_init_fe_1 },
+        {}
 };
 
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
 struct nouveau_oclass *
-nvc3_graph_oclass = &(struct nvc0_graph_oclass) {
+nvc4_graph_oclass = &(struct nvc0_graph_oclass) {
         .base.handle = NV_ENGINE(GR, 0xc3),
         .base.ofuncs = &(struct nouveau_ofuncs) {
                 .ctor = nvc0_graph_ctor,
@@ -102,9 +120,9 @@ nvc3_graph_oclass = &(struct nvc0_graph_oclass) {
                 .init = nvc0_graph_init,
                 .fini = _nouveau_graph_fini,
         },
-        .cclass = &nvc3_grctx_oclass,
+        .cclass = &nvc4_grctx_oclass,
         .sclass = nvc0_graph_sclass,
-        .mmio = nvc3_graph_init_mmio,
+        .mmio = nvc4_graph_pack_mmio,
         .fecs.ucode = &nvc0_graph_fecs_ucode,
         .gpccs.ucode = &nvc0_graph_gpccs_ucode,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvc8.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvc8.c
index 02845e567314..a6bf783e1256 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvc8.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvc8.c
@@ -23,6 +23,7 @@
  */
 
 #include "nvc0.h"
+#include "ctxnvc0.h"
 
 /*******************************************************************************
  * Graphics object classes
@@ -40,58 +41,11 @@ nvc8_graph_sclass[] = {
 };
 
 /*******************************************************************************
- * PGRAPH engine/subdev functions
+ * PGRAPH register lists
  ******************************************************************************/
 
-static struct nvc0_graph_init
-nvc8_graph_init_gpc[] = {
-        { 0x4184a0,   1, 0x04, 0x00000000 },
-        { 0x418604,   1, 0x04, 0x00000000 },
-        { 0x418680,   1, 0x04, 0x00000000 },
-        { 0x418714,   1, 0x04, 0x80000000 },
-        { 0x418384,   1, 0x04, 0x00000000 },
-        { 0x418814,   3, 0x04, 0x00000000 },
-        { 0x418b04,   1, 0x04, 0x00000000 },
-        { 0x4188c8,   2, 0x04, 0x00000000 },
-        { 0x4188d0,   1, 0x04, 0x00010000 },
-        { 0x4188d4,   1, 0x04, 0x00000001 },
-        { 0x418910,   1, 0x04, 0x00010001 },
-        { 0x418914,   1, 0x04, 0x00000301 },
-        { 0x418918,   1, 0x04, 0x00800000 },
-        { 0x418980,   1, 0x04, 0x77777770 },
-        { 0x418984,   3, 0x04, 0x77777777 },
-        { 0x418c04,   1, 0x04, 0x00000000 },
-        { 0x418c88,   1, 0x04, 0x00000000 },
-        { 0x418d00,   1, 0x04, 0x00000000 },
-        { 0x418f08,   1, 0x04, 0x00000000 },
-        { 0x418e00,   1, 0x04, 0x00000050 },
-        { 0x418e08,   1, 0x04, 0x00000000 },
-        { 0x41900c,   1, 0x04, 0x00000000 },
-        { 0x419018,   1, 0x04, 0x00000000 },
-        {}
-};
-
-static struct nvc0_graph_init
-nvc8_graph_init_tpc[] = {
-        { 0x419d08,   2, 0x04, 0x00000000 },
-        { 0x419d10,   1, 0x04, 0x00000014 },
-        { 0x419ab0,   1, 0x04, 0x00000000 },
-        { 0x419ab8,   1, 0x04, 0x000000e7 },
-        { 0x419abc,   2, 0x04, 0x00000000 },
-        { 0x41980c,   3, 0x04, 0x00000000 },
-        { 0x419844,   1, 0x04, 0x00000000 },
-        { 0x41984c,   1, 0x04, 0x00005bc5 },
-        { 0x419850,   4, 0x04, 0x00000000 },
-        { 0x419c98,   1, 0x04, 0x00000000 },
-        { 0x419ca8,   1, 0x04, 0x80000000 },
-        { 0x419cb4,   1, 0x04, 0x00000000 },
-        { 0x419cb8,   1, 0x04, 0x00008bf4 },
-        { 0x419cbc,   1, 0x04, 0x28137606 },
-        { 0x419cc0,   2, 0x04, 0x00000000 },
-        { 0x419bd4,   1, 0x04, 0x00800000 },
-        { 0x419bdc,   1, 0x04, 0x00000000 },
-        { 0x419d2c,   1, 0x04, 0x00000000 },
-        { 0x419c0c,   1, 0x04, 0x00000000 },
+static const struct nvc0_graph_init
+nvc8_graph_init_sm_0[] = {
         { 0x419e00,   1, 0x04, 0x00000000 },
         { 0x419ea0,   1, 0x04, 0x00000000 },
         { 0x419ea4,   1, 0x04, 0x00000100 },
@@ -108,22 +62,42 @@ nvc8_graph_init_tpc[] = {
         {}
 };
 
-static struct nvc0_graph_init *
-nvc8_graph_init_mmio[] = {
-        nvc0_graph_init_regs,
-        nvc0_graph_init_unk40xx,
-        nvc0_graph_init_unk44xx,
-        nvc0_graph_init_unk78xx,
-        nvc0_graph_init_unk60xx,
-        nvc0_graph_init_unk58xx,
-        nvc0_graph_init_unk80xx,
-        nvc8_graph_init_gpc,
-        nvc8_graph_init_tpc,
-        nvc0_graph_init_unk88xx,
-        nvc0_graph_tpc_0,
-        NULL
+static const struct nvc0_graph_pack
+nvc8_graph_pack_mmio[] = {
+        { nvc0_graph_init_main_0 },
+        { nvc0_graph_init_fe_0 },
+        { nvc0_graph_init_pri_0 },
+        { nvc0_graph_init_rstr2d_0 },
+        { nvc0_graph_init_pd_0 },
+        { nvc0_graph_init_ds_0 },
+        { nvc0_graph_init_scc_0 },
+        { nvc0_graph_init_prop_0 },
+        { nvc0_graph_init_gpc_unk_0 },
+        { nvc0_graph_init_setup_0 },
+        { nvc0_graph_init_crstr_0 },
+        { nvc1_graph_init_setup_1 },
+        { nvc0_graph_init_zcull_0 },
+        { nvc0_graph_init_gpm_0 },
+        { nvc0_graph_init_gpc_unk_1 },
+        { nvc0_graph_init_gcc_0 },
+        { nvc0_graph_init_tpccs_0 },
+        { nvc0_graph_init_tex_0 },
+        { nvc0_graph_init_pe_0 },
+        { nvc0_graph_init_l1c_0 },
+        { nvc0_graph_init_wwdx_0 },
+        { nvc0_graph_init_tpccs_1 },
+        { nvc0_graph_init_mpc_0 },
+        { nvc8_graph_init_sm_0 },
+        { nvc0_graph_init_be_0 },
+        { nvc0_graph_init_fe_1 },
+        { nvc0_graph_init_pe_1 },
+        {}
 };
 
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
 struct nouveau_oclass *
 nvc8_graph_oclass = &(struct nvc0_graph_oclass) {
         .base.handle = NV_ENGINE(GR, 0xc8),
@@ -135,7 +109,7 @@ nvc8_graph_oclass = &(struct nvc0_graph_oclass) {
         },
         .cclass = &nvc8_grctx_oclass,
         .sclass = nvc8_graph_sclass,
-        .mmio = nvc8_graph_init_mmio,
+        .mmio = nvc8_graph_pack_mmio,
         .fecs.ucode = &nvc0_graph_fecs_ucode,
         .gpccs.ucode = &nvc0_graph_gpccs_ucode,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvd7.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvd7.c
index 5052d7ab4d72..2a6a94e2a041 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvd7.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvd7.c
@@ -23,6 +23,77 @@
  */
 
 #include "nvc0.h"
+#include "ctxnvc0.h"
+
+/*******************************************************************************
+ * PGRAPH register lists
+ ******************************************************************************/
+
+static const struct nvc0_graph_init
+nvd7_graph_init_pe_0[] = {
+        { 0x41980c,   1, 0x04, 0x00000010 },
+        { 0x419844,   1, 0x04, 0x00000000 },
+        { 0x41984c,   1, 0x04, 0x00005bc8 },
+        { 0x419850,   3, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvd7_graph_init_pes_0[] = {
+        { 0x41be04,   1, 0x04, 0x00000000 },
+        { 0x41be08,   1, 0x04, 0x00000004 },
+        { 0x41be0c,   1, 0x04, 0x00000000 },
+        { 0x41be10,   1, 0x04, 0x003b8bc7 },
+        { 0x41be14,   2, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvd7_graph_init_wwdx_0[] = {
+        { 0x41bfd4,   1, 0x04, 0x00800000 },
+        { 0x41bfdc,   1, 0x04, 0x00000000 },
+        { 0x41bff8,   2, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvd7_graph_init_cbm_0[] = {
+        { 0x41becc,   1, 0x04, 0x00000000 },
+        { 0x41bee8,   2, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_pack
+nvd7_graph_pack_mmio[] = {
+        { nvc0_graph_init_main_0 },
+        { nvc0_graph_init_fe_0 },
+        { nvc0_graph_init_pri_0 },
+        { nvc0_graph_init_rstr2d_0 },
+        { nvd9_graph_init_pd_0 },
+        { nvd9_graph_init_ds_0 },
+        { nvc0_graph_init_scc_0 },
+        { nvd9_graph_init_prop_0 },
+        { nvc1_graph_init_gpc_unk_0 },
+        { nvc0_graph_init_setup_0 },
+        { nvc0_graph_init_crstr_0 },
+        { nvc1_graph_init_setup_1 },
+        { nvc0_graph_init_zcull_0 },
+        { nvd9_graph_init_gpm_0 },
+        { nvd9_graph_init_gpc_unk_1 },
+        { nvc0_graph_init_gcc_0 },
+        { nvc0_graph_init_tpccs_0 },
+        { nvd9_graph_init_tex_0 },
+        { nvd7_graph_init_pe_0 },
+        { nvc0_graph_init_l1c_0 },
+        { nvc0_graph_init_mpc_0 },
+        { nvd9_graph_init_sm_0 },
+        { nvd7_graph_init_pes_0 },
+        { nvd7_graph_init_wwdx_0 },
+        { nvd7_graph_init_cbm_0 },
+        { nvc0_graph_init_be_0 },
+        { nvd9_graph_init_fe_1 },
+        {}
+};
 
 /*******************************************************************************
  * PGRAPH engine/subdev functions
@@ -48,108 +119,6 @@ nvd7_graph_gpccs_ucode = {
         .data.size = sizeof(nvd7_grgpc_data),
 };
 
-static struct nvc0_graph_init
-nvd7_graph_init_gpc[] = {
-        { 0x418408,   1, 0x04, 0x00000000 },
-        { 0x4184a0,   1, 0x04, 0x00000000 },
-        { 0x4184a4,   2, 0x04, 0x00000000 },
-        { 0x418604,   1, 0x04, 0x00000000 },
-        { 0x418680,   1, 0x04, 0x00000000 },
-        { 0x418714,   1, 0x04, 0x00000000 },
-        { 0x418384,   1, 0x04, 0x00000000 },
-        { 0x418814,   3, 0x04, 0x00000000 },
-        { 0x418b04,   1, 0x04, 0x00000000 },
-        { 0x4188c8,   2, 0x04, 0x00000000 },
-        { 0x4188d0,   1, 0x04, 0x00010000 },
-        { 0x4188d4,   1, 0x04, 0x00000001 },
-        { 0x418910,   1, 0x04, 0x00010001 },
-        { 0x418914,   1, 0x04, 0x00000301 },
-        { 0x418918,   1, 0x04, 0x00800000 },
-        { 0x418980,   1, 0x04, 0x77777770 },
-        { 0x418984,   3, 0x04, 0x77777777 },
-        { 0x418c04,   1, 0x04, 0x00000000 },
-        { 0x418c64,   1, 0x04, 0x00000000 },
-        { 0x418c68,   1, 0x04, 0x00000000 },
-        { 0x418c88,   1, 0x04, 0x00000000 },
-        { 0x418cb4,   2, 0x04, 0x00000000 },
-        { 0x418d00,   1, 0x04, 0x00000000 },
-        { 0x418d28,   1, 0x04, 0x00000000 },
-        { 0x418f00,   1, 0x04, 0x00000000 },
-        { 0x418f08,   1, 0x04, 0x00000000 },
-        { 0x418f20,   2, 0x04, 0x00000000 },
-        { 0x418e00,   1, 0x04, 0x00000003 },
-        { 0x418e08,   1, 0x04, 0x00000000 },
-        { 0x418e1c,   1, 0x04, 0x00000000 },
-        { 0x418e20,   1, 0x04, 0x00000000 },
-        { 0x41900c,   1, 0x04, 0x00000000 },
-        { 0x419018,   1, 0x04, 0x00000000 },
-        {}
-};
-
-static struct nvc0_graph_init
-nvd7_graph_init_tpc[] = {
-        { 0x419d08,   2, 0x04, 0x00000000 },
-        { 0x419d10,   1, 0x04, 0x00000014 },
-        { 0x419ab0,   1, 0x04, 0x00000000 },
-        { 0x419ac8,   1, 0x04, 0x00000000 },
-        { 0x419ab8,   1, 0x04, 0x000000e7 },
-        { 0x419abc,   2, 0x04, 0x00000000 },
-        { 0x419ab4,   1, 0x04, 0x00000000 },
-        { 0x41980c,   1, 0x04, 0x00000010 },
-        { 0x419844,   1, 0x04, 0x00000000 },
-        { 0x41984c,   1, 0x04, 0x00005bc8 },
-        { 0x419850,   2, 0x04, 0x00000000 },
-        { 0x419c98,   1, 0x04, 0x00000000 },
-        { 0x419ca8,   1, 0x04, 0x80000000 },
-        { 0x419cb4,   1, 0x04, 0x00000000 },
-        { 0x419cb8,   1, 0x04, 0x00008bf4 },
-        { 0x419cbc,   1, 0x04, 0x28137606 },
-        { 0x419cc0,   2, 0x04, 0x00000000 },
-        { 0x419c0c,   1, 0x04, 0x00000000 },
-        { 0x419e00,   1, 0x04, 0x00000000 },
-        { 0x419ea0,   1, 0x04, 0x00000000 },
-        { 0x419ea4,   1, 0x04, 0x00000100 },
-        { 0x419ea8,   1, 0x04, 0x02001100 },
-        { 0x419eac,   1, 0x04, 0x11100702 },
-        { 0x419eb0,   1, 0x04, 0x00000003 },
-        { 0x419eb4,   4, 0x04, 0x00000000 },
-        { 0x419ec8,   1, 0x04, 0x0e063818 },
-        { 0x419ecc,   1, 0x04, 0x0e060e06 },
-        { 0x419ed0,   1, 0x04, 0x00003818 },
-        { 0x419ed4,   1, 0x04, 0x011104f1 },
-        { 0x419edc,   1, 0x04, 0x00000000 },
-        { 0x419f00,   1, 0x04, 0x00000000 },
-        { 0x419f2c,   1, 0x04, 0x00000000 },
-        {}
-};
-
-static struct nvc0_graph_init
-nvd7_graph_init_tpc_0[] = {
-        { 0x40402c,   1, 0x04, 0x00000000 },
-        { 0x4040f0,   1, 0x04, 0x00000000 },
-        { 0x404174,   1, 0x04, 0x00000000 },
-        { 0x503018,   1, 0x04, 0x00000001 },
-        {}
-};
-
-static struct nvc0_graph_init *
-nvd7_graph_init_mmio[] = {
-        nvc0_graph_init_regs,
-        nvc0_graph_init_unk40xx,
-        nvc0_graph_init_unk44xx,
-        nvc0_graph_init_unk78xx,
-        nvc0_graph_init_unk60xx,
-        nvd9_graph_init_unk64xx,
-        nvd9_graph_init_unk58xx,
-        nvc0_graph_init_unk80xx,
-        nvd7_graph_init_gpc,
-        nvd7_graph_init_tpc,
-        nve4_graph_init_unk,
-        nvc0_graph_init_unk88xx,
-        nvd7_graph_init_tpc_0,
-        NULL
-};
-
 struct nouveau_oclass *
 nvd7_graph_oclass = &(struct nvc0_graph_oclass) {
         .base.handle = NV_ENGINE(GR, 0xd7),
@@ -161,7 +130,7 @@ nvd7_graph_oclass = &(struct nvc0_graph_oclass) {
         },
         .cclass = &nvd7_grctx_oclass,
         .sclass = nvc8_graph_sclass,
-        .mmio = nvd7_graph_init_mmio,
+        .mmio = nvd7_graph_pack_mmio,
         .fecs.ucode = &nvd7_graph_fecs_ucode,
         .gpccs.ucode = &nvd7_graph_gpccs_ucode,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvd9.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvd9.c
index 652098e0df3f..00fdf202fb92 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvd9.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvd9.c
@@ -23,76 +23,70 @@
  */
 
 #include "nvc0.h"
+#include "ctxnvc0.h"
 
 /*******************************************************************************
- * PGRAPH engine/subdev functions
+ * PGRAPH register lists
  ******************************************************************************/
 
-struct nvc0_graph_init
-nvd9_graph_init_unk64xx[] = {
+const struct nvc0_graph_init
+nvd9_graph_init_pd_0[] = {
+        { 0x406024,   1, 0x04, 0x00000000 },
         { 0x4064f0,   3, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nvd9_graph_init_unk58xx[] = {
+const struct nvc0_graph_init
+nvd9_graph_init_ds_0[] = {
         { 0x405844,   1, 0x04, 0x00ffffff },
         { 0x405850,   1, 0x04, 0x00000000 },
         { 0x405900,   1, 0x04, 0x00002834 },
         { 0x405908,   1, 0x04, 0x00000000 },
-        { 0x405928,   1, 0x04, 0x00000000 },
-        { 0x40592c,   1, 0x04, 0x00000000 },
+        { 0x405928,   2, 0x04, 0x00000000 },
         {}
 };
 
-static struct nvc0_graph_init
-nvd9_graph_init_gpc[] = {
+const struct nvc0_graph_init
+nvd9_graph_init_prop_0[] = {
         { 0x418408,   1, 0x04, 0x00000000 },
-        { 0x4184a0,   1, 0x04, 0x00000000 },
-        { 0x4184a4,   2, 0x04, 0x00000000 },
-        { 0x418604,   1, 0x04, 0x00000000 },
-        { 0x418680,   1, 0x04, 0x00000000 },
-        { 0x418714,   1, 0x04, 0x00000000 },
-        { 0x418384,   1, 0x04, 0x00000000 },
-        { 0x418814,   3, 0x04, 0x00000000 },
-        { 0x418b04,   1, 0x04, 0x00000000 },
-        { 0x4188c8,   2, 0x04, 0x00000000 },
-        { 0x4188d0,   1, 0x04, 0x00010000 },
-        { 0x4188d4,   1, 0x04, 0x00000001 },
-        { 0x418910,   1, 0x04, 0x00010001 },
-        { 0x418914,   1, 0x04, 0x00000301 },
-        { 0x418918,   1, 0x04, 0x00800000 },
-        { 0x418980,   1, 0x04, 0x77777770 },
-        { 0x418984,   3, 0x04, 0x77777777 },
+        { 0x4184a0,   3, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvd9_graph_init_gpm_0[] = {
         { 0x418c04,   1, 0x04, 0x00000000 },
-        { 0x418c64,   1, 0x04, 0x00000000 },
-        { 0x418c68,   1, 0x04, 0x00000000 },
+        { 0x418c64,   2, 0x04, 0x00000000 },
         { 0x418c88,   1, 0x04, 0x00000000 },
         { 0x418cb4,   2, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvd9_graph_init_gpc_unk_1[] = {
         { 0x418d00,   1, 0x04, 0x00000000 },
-        { 0x418d28,   1, 0x04, 0x00000000 },
-        { 0x418d2c,   1, 0x04, 0x00000000 },
+        { 0x418d28,   2, 0x04, 0x00000000 },
         { 0x418f00,   1, 0x04, 0x00000000 },
         { 0x418f08,   1, 0x04, 0x00000000 },
         { 0x418f20,   2, 0x04, 0x00000000 },
         { 0x418e00,   1, 0x04, 0x00000003 },
         { 0x418e08,   1, 0x04, 0x00000000 },
-        { 0x418e1c,   1, 0x04, 0x00000000 },
-        { 0x418e20,   1, 0x04, 0x00000000 },
-        { 0x41900c,   1, 0x04, 0x00000000 },
-        { 0x419018,   1, 0x04, 0x00000000 },
+        { 0x418e1c,   2, 0x04, 0x00000000 },
         {}
 };
 
-static struct nvc0_graph_init
-nvd9_graph_init_tpc[] = {
-        { 0x419d08,   2, 0x04, 0x00000000 },
-        { 0x419d10,   1, 0x04, 0x00000014 },
+const struct nvc0_graph_init
+nvd9_graph_init_tex_0[] = {
         { 0x419ab0,   1, 0x04, 0x00000000 },
         { 0x419ac8,   1, 0x04, 0x00000000 },
         { 0x419ab8,   1, 0x04, 0x000000e7 },
         { 0x419abc,   2, 0x04, 0x00000000 },
         { 0x419ab4,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvd9_graph_init_pe_0[] = {
         { 0x41980c,   1, 0x04, 0x00000010 },
         { 0x419810,   1, 0x04, 0x00000000 },
         { 0x419814,   1, 0x04, 0x00000004 },
@@ -100,20 +94,26 @@ nvd9_graph_init_tpc[] = {
         { 0x41984c,   1, 0x04, 0x0000a918 },
         { 0x419850,   4, 0x04, 0x00000000 },
         { 0x419880,   1, 0x04, 0x00000002 },
-        { 0x419c98,   1, 0x04, 0x00000000 },
-        { 0x419ca8,   1, 0x04, 0x80000000 },
-        { 0x419cb4,   1, 0x04, 0x00000000 },
-        { 0x419cb8,   1, 0x04, 0x00008bf4 },
-        { 0x419cbc,   1, 0x04, 0x28137606 },
-        { 0x419cc0,   2, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvd9_graph_init_wwdx_0[] = {
         { 0x419bd4,   1, 0x04, 0x00800000 },
         { 0x419bdc,   1, 0x04, 0x00000000 },
-        { 0x419bf8,   1, 0x04, 0x00000000 },
-        { 0x419bfc,   1, 0x04, 0x00000000 },
+        { 0x419bf8,   2, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvd9_graph_init_tpccs_1[] = {
         { 0x419d2c,   1, 0x04, 0x00000000 },
-        { 0x419d48,   1, 0x04, 0x00000000 },
-        { 0x419d4c,   1, 0x04, 0x00000000 },
-        { 0x419c0c,   1, 0x04, 0x00000000 },
+        { 0x419d48,   2, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvd9_graph_init_sm_0[] = {
         { 0x419e00,   1, 0x04, 0x00000000 },
         { 0x419ea0,   1, 0x04, 0x00000000 },
         { 0x419ea4,   1, 0x04, 0x00000100 },
@@ -131,23 +131,49 @@ nvd9_graph_init_tpc[] = {
         {}
 };
 
-static struct nvc0_graph_init *
-nvd9_graph_init_mmio[] = {
-        nvc0_graph_init_regs,
-        nvc0_graph_init_unk40xx,
-        nvc0_graph_init_unk44xx,
-        nvc0_graph_init_unk78xx,
-        nvc0_graph_init_unk60xx,
-        nvd9_graph_init_unk64xx,
-        nvd9_graph_init_unk58xx,
-        nvc0_graph_init_unk80xx,
-        nvd9_graph_init_gpc,
-        nvd9_graph_init_tpc,
-        nvc0_graph_init_unk88xx,
-        nvc0_graph_tpc_0,
-        NULL
+const struct nvc0_graph_init
+nvd9_graph_init_fe_1[] = {
+        { 0x40402c,   1, 0x04, 0x00000000 },
+        { 0x4040f0,   1, 0x04, 0x00000000 },
+        { 0x404174,   1, 0x04, 0x00000000 },
+        {}
 };
 
+static const struct nvc0_graph_pack
+nvd9_graph_pack_mmio[] = {
+        { nvc0_graph_init_main_0 },
+        { nvc0_graph_init_fe_0 },
+        { nvc0_graph_init_pri_0 },
+        { nvc0_graph_init_rstr2d_0 },
+        { nvd9_graph_init_pd_0 },
+        { nvd9_graph_init_ds_0 },
+        { nvc0_graph_init_scc_0 },
+        { nvd9_graph_init_prop_0 },
+        { nvc1_graph_init_gpc_unk_0 },
+        { nvc0_graph_init_setup_0 },
+        { nvc0_graph_init_crstr_0 },
+        { nvc1_graph_init_setup_1 },
+        { nvc0_graph_init_zcull_0 },
+        { nvd9_graph_init_gpm_0 },
+        { nvd9_graph_init_gpc_unk_1 },
+        { nvc0_graph_init_gcc_0 },
+        { nvc0_graph_init_tpccs_0 },
+        { nvd9_graph_init_tex_0 },
+        { nvd9_graph_init_pe_0 },
+        { nvc0_graph_init_l1c_0 },
+        { nvd9_graph_init_wwdx_0 },
+        { nvd9_graph_init_tpccs_1 },
+        { nvc0_graph_init_mpc_0 },
+        { nvd9_graph_init_sm_0 },
+        { nvc0_graph_init_be_0 },
+        { nvd9_graph_init_fe_1 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
 struct nouveau_oclass *
 nvd9_graph_oclass = &(struct nvc0_graph_oclass) {
         .base.handle = NV_ENGINE(GR, 0xd9),
@@ -159,7 +185,7 @@ nvd9_graph_oclass = &(struct nvc0_graph_oclass) {
         },
         .cclass = &nvd9_grctx_oclass,
         .sclass = nvc8_graph_sclass,
-        .mmio = nvd9_graph_init_mmio,
+        .mmio = nvd9_graph_pack_mmio,
         .fecs.ucode = &nvc0_graph_fecs_ucode,
         .gpccs.ucode = &nvc0_graph_gpccs_ucode,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nve4.c b/drivers/gpu/drm/nouveau/core/engine/graph/nve4.c
index 05ec09c88517..f7c011217175 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nve4.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nve4.c
@@ -23,6 +23,7 @@
  */
 
 #include "nvc0.h"
+#include "ctxnvc0.h"
 
 /*******************************************************************************
  * Graphics object classes
@@ -38,11 +39,11 @@ nve4_graph_sclass[] = {
 };
 
 /*******************************************************************************
- * PGRAPH engine/subdev functions
+ * PGRAPH register lists
  ******************************************************************************/
 
-struct nvc0_graph_init
-nve4_graph_init_regs[] = {
+const struct nvc0_graph_init
+nve4_graph_init_main_0[] = {
         { 0x400080,   1, 0x04, 0x003083c2 },
         { 0x400088,   1, 0x04, 0x0001ffe7 },
         { 0x40008c,   1, 0x04, 0x00000000 },
@@ -57,81 +58,59 @@ nve4_graph_init_regs[] = {
         {}
 };
 
-static struct nvc0_graph_init
-nve4_graph_init_unk58xx[] = {
+static const struct nvc0_graph_init
+nve4_graph_init_ds_0[] = {
         { 0x405844,   1, 0x04, 0x00ffffff },
         { 0x405850,   1, 0x04, 0x00000000 },
         { 0x405900,   1, 0x04, 0x0000ff34 },
         { 0x405908,   1, 0x04, 0x00000000 },
-        { 0x405928,   1, 0x04, 0x00000000 },
-        { 0x40592c,   1, 0x04, 0x00000000 },
+        { 0x405928,   2, 0x04, 0x00000000 },
         {}
 };
 
-static struct nvc0_graph_init
-nve4_graph_init_unk70xx[] = {
+static const struct nvc0_graph_init
+nve4_graph_init_sked_0[] = {
         { 0x407010,   1, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nve4_graph_init_unk5bxx[] = {
+static const struct nvc0_graph_init
+nve4_graph_init_cwd_0[] = {
         { 0x405b50,   1, 0x04, 0x00000000 },
         {}
 };
 
-static struct nvc0_graph_init
-nve4_graph_init_gpc[] = {
-        { 0x418408,   1, 0x04, 0x00000000 },
-        { 0x4184a0,   1, 0x04, 0x00000000 },
-        { 0x4184a4,   2, 0x04, 0x00000000 },
-        { 0x418604,   1, 0x04, 0x00000000 },
-        { 0x418680,   1, 0x04, 0x00000000 },
-        { 0x418714,   1, 0x04, 0x00000000 },
-        { 0x418384,   1, 0x04, 0x00000000 },
-        { 0x418814,   3, 0x04, 0x00000000 },
-        { 0x418b04,   1, 0x04, 0x00000000 },
-        { 0x4188c8,   2, 0x04, 0x00000000 },
-        { 0x4188d0,   1, 0x04, 0x00010000 },
-        { 0x4188d4,   1, 0x04, 0x00000001 },
-        { 0x418910,   1, 0x04, 0x00010001 },
-        { 0x418914,   1, 0x04, 0x00000301 },
-        { 0x418918,   1, 0x04, 0x00800000 },
-        { 0x418980,   1, 0x04, 0x77777770 },
-        { 0x418984,   3, 0x04, 0x77777777 },
-        { 0x418c04,   1, 0x04, 0x00000000 },
-        { 0x418c64,   1, 0x04, 0x00000000 },
-        { 0x418c68,   1, 0x04, 0x00000000 },
-        { 0x418c88,   1, 0x04, 0x00000000 },
-        { 0x418cb4,   2, 0x04, 0x00000000 },
+static const struct nvc0_graph_init
+nve4_graph_init_gpc_unk_1[] = {
         { 0x418d00,   1, 0x04, 0x00000000 },
-        { 0x418d28,   1, 0x04, 0x00000000 },
-        { 0x418d2c,   1, 0x04, 0x00000000 },
+        { 0x418d28,   2, 0x04, 0x00000000 },
         { 0x418f00,   1, 0x04, 0x00000000 },
         { 0x418f08,   1, 0x04, 0x00000000 },
         { 0x418f20,   2, 0x04, 0x00000000 },
         { 0x418e00,   1, 0x04, 0x00000060 },
         { 0x418e08,   1, 0x04, 0x00000000 },
-        { 0x418e1c,   1, 0x04, 0x00000000 },
-        { 0x418e20,   1, 0x04, 0x00000000 },
-        { 0x41900c,   1, 0x04, 0x00000000 },
-        { 0x419018,   1, 0x04, 0x00000000 },
+        { 0x418e1c,   2, 0x04, 0x00000000 },
         {}
 };
 
-static struct nvc0_graph_init
-nve4_graph_init_tpc[] = {
+const struct nvc0_graph_init
+nve4_graph_init_tpccs_0[] = {
         { 0x419d0c,   1, 0x04, 0x00000000 },
         { 0x419d10,   1, 0x04, 0x00000014 },
-        { 0x419ab0,   1, 0x04, 0x00000000 },
-        { 0x419ac8,   1, 0x04, 0x00000000 },
-        { 0x419ab8,   1, 0x04, 0x000000e7 },
-        { 0x419abc,   2, 0x04, 0x00000000 },
-        { 0x419ab4,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nve4_graph_init_pe_0[] = {
         { 0x41980c,   1, 0x04, 0x00000010 },
         { 0x419844,   1, 0x04, 0x00000000 },
         { 0x419850,   1, 0x04, 0x00000004 },
         { 0x419854,   2, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nve4_graph_init_l1c_0[] = {
         { 0x419c98,   1, 0x04, 0x00000000 },
         { 0x419ca8,   1, 0x04, 0x00000000 },
         { 0x419cb0,   1, 0x04, 0x01000000 },
@@ -141,39 +120,25 @@ nve4_graph_init_tpc[] = {
         { 0x419cbc,   1, 0x04, 0x28137646 },
         { 0x419cc0,   2, 0x04, 0x00000000 },
         { 0x419c80,   1, 0x04, 0x00020232 },
-        { 0x419c0c,   1, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nve4_graph_init_sm_0[] = {
         { 0x419e00,   1, 0x04, 0x00000000 },
         { 0x419ea0,   1, 0x04, 0x00000000 },
         { 0x419ee4,   1, 0x04, 0x00000000 },
         { 0x419ea4,   1, 0x04, 0x00000100 },
         { 0x419ea8,   1, 0x04, 0x00000000 },
-        { 0x419eb4,   1, 0x04, 0x00000000 },
-        { 0x419eb8,   3, 0x04, 0x00000000 },
+        { 0x419eb4,   4, 0x04, 0x00000000 },
         { 0x419edc,   1, 0x04, 0x00000000 },
         { 0x419f00,   1, 0x04, 0x00000000 },
         { 0x419f74,   1, 0x04, 0x00000555 },
         {}
 };
 
-struct nvc0_graph_init
-nve4_graph_init_unk[] = {
-        { 0x41be04,   1, 0x04, 0x00000000 },
-        { 0x41be08,   1, 0x04, 0x00000004 },
-        { 0x41be0c,   1, 0x04, 0x00000000 },
-        { 0x41be10,   1, 0x04, 0x003b8bc7 },
-        { 0x41be14,   2, 0x04, 0x00000000 },
-        { 0x41bfd4,   1, 0x04, 0x00800000 },
-        { 0x41bfdc,   1, 0x04, 0x00000000 },
-        { 0x41bff8,   1, 0x04, 0x00000000 },
-        { 0x41bffc,   1, 0x04, 0x00000000 },
-        { 0x41becc,   1, 0x04, 0x00000000 },
-        { 0x41bee8,   1, 0x04, 0x00000000 },
-        { 0x41beec,   1, 0x04, 0x00000000 },
-        {}
-};
-
-struct nvc0_graph_init
-nve4_graph_init_unk88xx[] = {
+const struct nvc0_graph_init
+nve4_graph_init_be_0[] = {
         { 0x40880c,   1, 0x04, 0x00000000 },
         { 0x408850,   1, 0x04, 0x00000004 },
         { 0x408910,   9, 0x04, 0x00000000 },
@@ -186,6 +151,67 @@ nve4_graph_init_unk88xx[] = {
         {}
 };
 
+static const struct nvc0_graph_pack
+nve4_graph_pack_mmio[] = {
+        { nve4_graph_init_main_0 },
+        { nvc0_graph_init_fe_0 },
+        { nvc0_graph_init_pri_0 },
+        { nvc0_graph_init_rstr2d_0 },
+        { nvd9_graph_init_pd_0 },
+        { nve4_graph_init_ds_0 },
+        { nvc0_graph_init_scc_0 },
+        { nve4_graph_init_sked_0 },
+        { nve4_graph_init_cwd_0 },
+        { nvd9_graph_init_prop_0 },
+        { nvc1_graph_init_gpc_unk_0 },
+        { nvc0_graph_init_setup_0 },
+        { nvc0_graph_init_crstr_0 },
+        { nvc1_graph_init_setup_1 },
+        { nvc0_graph_init_zcull_0 },
+        { nvd9_graph_init_gpm_0 },
+        { nve4_graph_init_gpc_unk_1 },
+        { nvc0_graph_init_gcc_0 },
+        { nve4_graph_init_tpccs_0 },
+        { nvd9_graph_init_tex_0 },
+        { nve4_graph_init_pe_0 },
+        { nve4_graph_init_l1c_0 },
+        { nvc0_graph_init_mpc_0 },
+        { nve4_graph_init_sm_0 },
+        { nvd7_graph_init_pes_0 },
+        { nvd7_graph_init_wwdx_0 },
+        { nvd7_graph_init_cbm_0 },
+        { nve4_graph_init_be_0 },
+        { nvc0_graph_init_fe_1 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+static int
+nve4_graph_fini(struct nouveau_object *object, bool suspend)
+{
+        struct nvc0_graph_priv *priv = (void *)object;
+
+        /*XXX: this is a nasty hack to power on gr on certain boards
+         *     where it's disabled by therm, somehow.  ideally it'd
+         *     be nice to know when we should be doing this, and why,
+         *     but, it's yet to be determined.  for now we test for
+         *     the particular mmio error that occurs in the situation,
+         *     and then bash therm in the way nvidia do.
+         */
+        nv_mask(priv, 0x000200, 0x08001000, 0x08001000);
+        nv_rd32(priv, 0x000200);
+        if (nv_rd32(priv, 0x400700) == 0xbadf1000) {
+                nv_mask(priv, 0x000200, 0x08001000, 0x00000000);
+                nv_rd32(priv, 0x000200);
+                nv_mask(priv, 0x020004, 0xc0000000, 0x40000000);
+        }
+
+        return nouveau_graph_fini(&priv->base, suspend);
+}
+
 int
 nve4_graph_init(struct nouveau_object *object)
 {
@@ -210,8 +236,7 @@ nve4_graph_init(struct nouveau_object *object)
         nv_wr32(priv, GPC_BCAST(0x08b4), priv->unk4188b4->addr >> 8);
         nv_wr32(priv, GPC_BCAST(0x08b8), priv->unk4188b8->addr >> 8);
 
-        for (i = 0; oclass->mmio[i]; i++)
-                nvc0_graph_mmio(priv, oclass->mmio[i]);
+        nvc0_graph_mmio(priv, oclass->mmio);
 
         nv_wr32(priv, GPC_UNIT(0, 0x3018), 0x00000001);
 
@@ -298,25 +323,6 @@ nve4_graph_init(struct nouveau_object *object)
         return nvc0_graph_init_ctxctl(priv);
 }
 
-static struct nvc0_graph_init *
-nve4_graph_init_mmio[] = {
-        nve4_graph_init_regs,
-        nvc0_graph_init_unk40xx,
-        nvc0_graph_init_unk44xx,
-        nvc0_graph_init_unk78xx,
-        nvc0_graph_init_unk60xx,
-        nvd9_graph_init_unk64xx,
-        nve4_graph_init_unk58xx,
-        nvc0_graph_init_unk80xx,
-        nve4_graph_init_unk70xx,
-        nve4_graph_init_unk5bxx,
-        nve4_graph_init_gpc,
-        nve4_graph_init_tpc,
-        nve4_graph_init_unk,
-        nve4_graph_init_unk88xx,
-        NULL
-};
-
 #include "fuc/hubnve0.fuc.h"
 
 static struct nvc0_graph_ucode
@@ -344,11 +350,11 @@ nve4_graph_oclass = &(struct nvc0_graph_oclass) {
                 .ctor = nvc0_graph_ctor,
                 .dtor = nvc0_graph_dtor,
                 .init = nve4_graph_init,
-                .fini = _nouveau_graph_fini,
+                .fini = nve4_graph_fini,
         },
         .cclass = &nve4_grctx_oclass,
         .sclass = nve4_graph_sclass,
-        .mmio = nve4_graph_init_mmio,
+        .mmio = nve4_graph_pack_mmio,
         .fecs.ucode = &nve4_graph_fecs_ucode,
         .gpccs.ucode = &nve4_graph_gpccs_ucode,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/graph/nvf0.c b/drivers/gpu/drm/nouveau/core/engine/graph/nvf0.c
index b1acb9939d95..c96762122b9b 100644
--- a/drivers/gpu/drm/nouveau/core/engine/graph/nvf0.c
+++ b/drivers/gpu/drm/nouveau/core/engine/graph/nvf0.c
@@ -23,6 +23,7 @@
  */
 
 #include "nvc0.h"
+#include "ctxnvc0.h"
 
 /*******************************************************************************
  * Graphics object classes
@@ -38,86 +39,57 @@ nvf0_graph_sclass[] = {
 };
 
 /*******************************************************************************
- * PGRAPH engine/subdev functions
+ * PGRAPH register lists
  ******************************************************************************/
 
-struct nvc0_graph_init
-nvf0_graph_init_unk40xx[] = {
+const struct nvc0_graph_init
+nvf0_graph_init_fe_0[] = {
         { 0x40415c,   1, 0x04, 0x00000000 },
         { 0x404170,   1, 0x04, 0x00000000 },
         { 0x4041b4,   1, 0x04, 0x00000000 },
         {}
 };
 
-static struct nvc0_graph_init
-nvf0_graph_init_unk58xx[] = {
+static const struct nvc0_graph_init
+nvf0_graph_init_ds_0[] = {
         { 0x405844,   1, 0x04, 0x00ffffff },
         { 0x405850,   1, 0x04, 0x00000000 },
         { 0x405900,   1, 0x04, 0x0000ff00 },
         { 0x405908,   1, 0x04, 0x00000000 },
-        { 0x405928,   1, 0x04, 0x00000000 },
-        { 0x40592c,   1, 0x04, 0x00000000 },
+        { 0x405928,   2, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nvf0_graph_init_unk70xx[] = {
+const struct nvc0_graph_init
+nvf0_graph_init_sked_0[] = {
         { 0x407010,   1, 0x04, 0x00000000 },
         { 0x407040,   1, 0x04, 0x80440424 },
         { 0x407048,   1, 0x04, 0x0000000a },
         {}
 };
 
-struct nvc0_graph_init
-nvf0_graph_init_unk5bxx[] = {
+const struct nvc0_graph_init
+nvf0_graph_init_cwd_0[] = {
         { 0x405b44,   1, 0x04, 0x00000000 },
         { 0x405b50,   1, 0x04, 0x00000000 },
         {}
 };
 
-static struct nvc0_graph_init
-nvf0_graph_init_gpc[] = {
-        { 0x418408,   1, 0x04, 0x00000000 },
-        { 0x4184a0,   1, 0x04, 0x00000000 },
-        { 0x4184a4,   2, 0x04, 0x00000000 },
-        { 0x418604,   1, 0x04, 0x00000000 },
-        { 0x418680,   1, 0x04, 0x00000000 },
-        { 0x418714,   1, 0x04, 0x00000000 },
-        { 0x418384,   1, 0x04, 0x00000000 },
-        { 0x418814,   3, 0x04, 0x00000000 },
-        { 0x418b04,   1, 0x04, 0x00000000 },
-        { 0x4188c8,   2, 0x04, 0x00000000 },
-        { 0x4188d0,   1, 0x04, 0x00010000 },
-        { 0x4188d4,   1, 0x04, 0x00000001 },
-        { 0x418910,   1, 0x04, 0x00010001 },
-        { 0x418914,   1, 0x04, 0x00000301 },
-        { 0x418918,   1, 0x04, 0x00800000 },
-        { 0x418980,   1, 0x04, 0x77777770 },
-        { 0x418984,   3, 0x04, 0x77777777 },
-        { 0x418c04,   1, 0x04, 0x00000000 },
-        { 0x418c64,   1, 0x04, 0x00000000 },
-        { 0x418c68,   1, 0x04, 0x00000000 },
-        { 0x418c88,   1, 0x04, 0x00000000 },
-        { 0x418cb4,   2, 0x04, 0x00000000 },
+const struct nvc0_graph_init
+nvf0_graph_init_gpc_unk_1[] = {
         { 0x418d00,   1, 0x04, 0x00000000 },
-        { 0x418d28,   1, 0x04, 0x00000000 },
-        { 0x418d2c,   1, 0x04, 0x00000000 },
+        { 0x418d28,   2, 0x04, 0x00000000 },
         { 0x418f00,   1, 0x04, 0x00000400 },
         { 0x418f08,   1, 0x04, 0x00000000 },
-        { 0x418f20,   1, 0x04, 0x00000000 },
-        { 0x418f24,   1, 0x04, 0x00000000 },
+        { 0x418f20,   2, 0x04, 0x00000000 },
         { 0x418e00,   1, 0x04, 0x00000000 },
         { 0x418e08,   1, 0x04, 0x00000000 },
         { 0x418e1c,   2, 0x04, 0x00000000 },
-        { 0x41900c,   1, 0x04, 0x00000000 },
-        { 0x419018,   1, 0x04, 0x00000000 },
         {}
 };
 
-struct nvc0_graph_init
-nvf0_graph_init_tpc[] = {
-        { 0x419d0c,   1, 0x04, 0x00000000 },
-        { 0x419d10,   1, 0x04, 0x00000014 },
+static const struct nvc0_graph_init
+nvf0_graph_init_tex_0[] = {
         { 0x419ab0,   1, 0x04, 0x00000000 },
         { 0x419ac8,   1, 0x04, 0x00000000 },
         { 0x419ab8,   1, 0x04, 0x000000e7 },
@@ -125,10 +97,11 @@ nvf0_graph_init_tpc[] = {
         { 0x419abc,   2, 0x04, 0x00000000 },
         { 0x419ab4,   1, 0x04, 0x00000000 },
         { 0x419aa8,   2, 0x04, 0x00000000 },
-        { 0x41980c,   1, 0x04, 0x00000010 },
-        { 0x419844,   1, 0x04, 0x00000000 },
-        { 0x419850,   1, 0x04, 0x00000004 },
-        { 0x419854,   2, 0x04, 0x00000000 },
+        {}
+};
+
+static const struct nvc0_graph_init
+nvf0_graph_init_l1c_0[] = {
         { 0x419c98,   1, 0x04, 0x00000000 },
         { 0x419ca8,   1, 0x04, 0x00000000 },
         { 0x419cb0,   1, 0x04, 0x01000000 },
@@ -139,7 +112,11 @@ nvf0_graph_init_tpc[] = {
         { 0x419cc0,   2, 0x04, 0x00000000 },
         { 0x419c80,   1, 0x04, 0x00020230 },
         { 0x419ccc,   2, 0x04, 0x00000000 },
-        { 0x419c0c,   1, 0x04, 0x00000000 },
+        {}
+};
+
+const struct nvc0_graph_init
+nvf0_graph_init_sm_0[] = {
         { 0x419e00,   1, 0x04, 0x00000080 },
         { 0x419ea0,   1, 0x04, 0x00000000 },
         { 0x419ee4,   1, 0x04, 0x00000000 },
@@ -155,6 +132,44 @@ nvf0_graph_init_tpc[] = {
         {}
 };
 
+static const struct nvc0_graph_pack
+nvf0_graph_pack_mmio[] = {
+        { nve4_graph_init_main_0 },
+        { nvf0_graph_init_fe_0 },
+        { nvc0_graph_init_pri_0 },
+        { nvc0_graph_init_rstr2d_0 },
+        { nvd9_graph_init_pd_0 },
+        { nvf0_graph_init_ds_0 },
+        { nvc0_graph_init_scc_0 },
+        { nvf0_graph_init_sked_0 },
+        { nvf0_graph_init_cwd_0 },
+        { nvd9_graph_init_prop_0 },
+        { nvc1_graph_init_gpc_unk_0 },
+        { nvc0_graph_init_setup_0 },
+        { nvc0_graph_init_crstr_0 },
+        { nvc1_graph_init_setup_1 },
+        { nvc0_graph_init_zcull_0 },
+        { nvd9_graph_init_gpm_0 },
+        { nvf0_graph_init_gpc_unk_1 },
+        { nvc0_graph_init_gcc_0 },
+        { nve4_graph_init_tpccs_0 },
+        { nvf0_graph_init_tex_0 },
+        { nve4_graph_init_pe_0 },
+        { nvf0_graph_init_l1c_0 },
+        { nvc0_graph_init_mpc_0 },
+        { nvf0_graph_init_sm_0 },
+        { nvd7_graph_init_pes_0 },
+        { nvd7_graph_init_wwdx_0 },
+        { nvd7_graph_init_cbm_0 },
+        { nve4_graph_init_be_0 },
+        { nvc0_graph_init_fe_1 },
+        {}
+};
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
 static int
 nvf0_graph_fini(struct nouveau_object *object, bool suspend)
 {
@@ -192,25 +207,6 @@ nvf0_graph_fini(struct nouveau_object *object, bool suspend)
         return nouveau_graph_fini(&priv->base, suspend);
 }
 
-static struct nvc0_graph_init *
-nvf0_graph_init_mmio[] = {
-        nve4_graph_init_regs,
-        nvf0_graph_init_unk40xx,
-        nvc0_graph_init_unk44xx,
-        nvc0_graph_init_unk78xx,
-        nvc0_graph_init_unk60xx,
-        nvd9_graph_init_unk64xx,
-        nvf0_graph_init_unk58xx,
-        nvc0_graph_init_unk80xx,
-        nvf0_graph_init_unk70xx,
-        nvf0_graph_init_unk5bxx,
-        nvf0_graph_init_gpc,
-        nvf0_graph_init_tpc,
-        nve4_graph_init_unk,
-        nve4_graph_init_unk88xx,
-        NULL
-};
-
 #include "fuc/hubnvf0.fuc.h"
 
 static struct nvc0_graph_ucode
@@ -242,7 +238,7 @@ nvf0_graph_oclass = &(struct nvc0_graph_oclass) {
         },
         .cclass = &nvf0_grctx_oclass,
         .sclass =  nvf0_graph_sclass,
-        .mmio = nvf0_graph_init_mmio,
+        .mmio = nvf0_graph_pack_mmio,
         .fecs.ucode = &nvf0_graph_fecs_ucode,
         .gpccs.ucode = &nvf0_graph_gpccs_ucode,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/engine/xtensa.c b/drivers/gpu/drm/nouveau/core/engine/xtensa.c
index 5f6ede7c4892..92384759d2f5 100644
--- a/drivers/gpu/drm/nouveau/core/engine/xtensa.c
+++ b/drivers/gpu/drm/nouveau/core/engine/xtensa.c
@@ -112,7 +112,7 @@ _nouveau_xtensa_init(struct nouveau_object *object)
                 snprintf(name, sizeof(name), "nouveau/nv84_xuc%03x",
                          xtensa->addr >> 12);
 
-                ret = request_firmware(&fw, name, &device->pdev->dev);
+                ret = request_firmware(&fw, name, nv_device_base(device));
                 if (ret) {
                         nv_warn(xtensa, "unable to load firmware %s\n", name);
                         return ret;
diff --git a/drivers/gpu/drm/nouveau/core/include/core/class.h b/drivers/gpu/drm/nouveau/core/include/core/class.h
index e71a4325e670..9c0cd73462d9 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/class.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/class.h
@@ -258,6 +258,7 @@ struct nv04_display_scanoutpos {
  * 9070: NVD0_DISP
  * 9170: NVE0_DISP
  * 9270: NVF0_DISP
+ * 9470: GM107_DISP
  */
 
 #define NV50_DISP_CLASS                                              0x00005070
@@ -268,6 +269,7 @@ struct nv04_display_scanoutpos {
 #define NVD0_DISP_CLASS                                              0x00009070
 #define NVE0_DISP_CLASS                                              0x00009170
 #define NVF0_DISP_CLASS                                              0x00009270
+#define GM107_DISP_CLASS                                             0x00009470
 
 #define NV50_DISP_MTHD                                               0x00000000
 #define NV50_DISP_MTHD_HEAD                                          0x00000003
@@ -342,6 +344,7 @@ struct nv50_display_class {
  * 907a: NVD0_DISP_CURS
  * 917a: NVE0_DISP_CURS
  * 927a: NVF0_DISP_CURS
+ * 947a: GM107_DISP_CURS
  */
 
 #define NV50_DISP_CURS_CLASS                                         0x0000507a
@@ -352,6 +355,7 @@ struct nv50_display_class {
 #define NVD0_DISP_CURS_CLASS                                         0x0000907a
 #define NVE0_DISP_CURS_CLASS                                         0x0000917a
 #define NVF0_DISP_CURS_CLASS                                         0x0000927a
+#define GM107_DISP_CURS_CLASS                                        0x0000947a
 
 struct nv50_display_curs_class {
         u32 head;
@@ -365,6 +369,7 @@ struct nv50_display_curs_class {
  * 907b: NVD0_DISP_OIMM
  * 917b: NVE0_DISP_OIMM
  * 927b: NVE0_DISP_OIMM
+ * 947b: GM107_DISP_OIMM
  */
 
 #define NV50_DISP_OIMM_CLASS                                         0x0000507b
@@ -375,6 +380,7 @@ struct nv50_display_curs_class {
 #define NVD0_DISP_OIMM_CLASS                                         0x0000907b
 #define NVE0_DISP_OIMM_CLASS                                         0x0000917b
 #define NVF0_DISP_OIMM_CLASS                                         0x0000927b
+#define GM107_DISP_OIMM_CLASS                                        0x0000947b
 
 struct nv50_display_oimm_class {
         u32 head;
@@ -388,6 +394,7 @@ struct nv50_display_oimm_class {
  * 907c: NVD0_DISP_SYNC
  * 917c: NVE0_DISP_SYNC
  * 927c: NVF0_DISP_SYNC
+ * 947c: GM107_DISP_SYNC
  */
 
 #define NV50_DISP_SYNC_CLASS                                         0x0000507c
@@ -398,6 +405,7 @@ struct nv50_display_oimm_class {
 #define NVD0_DISP_SYNC_CLASS                                         0x0000907c
 #define NVE0_DISP_SYNC_CLASS                                         0x0000917c
 #define NVF0_DISP_SYNC_CLASS                                         0x0000927c
+#define GM107_DISP_SYNC_CLASS                                        0x0000947c
 
 struct nv50_display_sync_class {
         u32 pushbuf;
@@ -412,6 +420,7 @@ struct nv50_display_sync_class {
  * 907d: NVD0_DISP_MAST
  * 917d: NVE0_DISP_MAST
  * 927d: NVF0_DISP_MAST
+ * 947d: GM107_DISP_MAST
  */
 
 #define NV50_DISP_MAST_CLASS                                         0x0000507d
@@ -422,6 +431,7 @@ struct nv50_display_sync_class {
 #define NVD0_DISP_MAST_CLASS                                         0x0000907d
 #define NVE0_DISP_MAST_CLASS                                         0x0000917d
 #define NVF0_DISP_MAST_CLASS                                         0x0000927d
+#define GM107_DISP_MAST_CLASS                                        0x0000947d
 
 struct nv50_display_mast_class {
         u32 pushbuf;
@@ -435,6 +445,7 @@ struct nv50_display_mast_class {
  * 907e: NVD0_DISP_OVLY
  * 917e: NVE0_DISP_OVLY
  * 927e: NVF0_DISP_OVLY
+ * 947e: GM107_DISP_OVLY
  */
 
 #define NV50_DISP_OVLY_CLASS                                         0x0000507e
@@ -445,6 +456,7 @@ struct nv50_display_mast_class {
 #define NVD0_DISP_OVLY_CLASS                                         0x0000907e
 #define NVE0_DISP_OVLY_CLASS                                         0x0000917e
 #define NVF0_DISP_OVLY_CLASS                                         0x0000927e
+#define GM107_DISP_OVLY_CLASS                                        0x0000947e
 
 struct nv50_display_ovly_class {
         u32 pushbuf;
diff --git a/drivers/gpu/drm/nouveau/core/include/core/device.h b/drivers/gpu/drm/nouveau/core/include/core/device.h
index 7b8ea221b00d..a8a9a9cf16cb 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/device.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/device.h
@@ -40,6 +40,7 @@ enum nv_subdev_type {
 
         NVDEV_ENGINE_FIRST,
         NVDEV_ENGINE_DMAOBJ = NVDEV_ENGINE_FIRST,
+        NVDEV_ENGINE_IFB,
         NVDEV_ENGINE_FIFO,
         NVDEV_ENGINE_SW,
         NVDEV_ENGINE_GR,
@@ -65,6 +66,7 @@ struct nouveau_device {
         struct list_head head;
 
         struct pci_dev *pdev;
+        struct platform_device *platformdev;
         u64 handle;
 
         const char *cfgopt;
@@ -84,6 +86,7 @@ struct nouveau_device {
                 NV_C0    = 0xc0,
                 NV_D0    = 0xd0,
                 NV_E0    = 0xe0,
+                GM100    = 0x110,
         } card_type;
         u32 chipset;
         u32 crystal;
@@ -140,4 +143,32 @@ nv_device_match(struct nouveau_object *object, u16 dev, u16 ven, u16 sub)
                device->pdev->subsystem_device == sub;
 }
 
+static inline bool
+nv_device_is_pci(struct nouveau_device *device)
+{
+        return device->pdev != NULL;
+}
+
+static inline struct device *
+nv_device_base(struct nouveau_device *device)
+{
+        return nv_device_is_pci(device) ? &device->pdev->dev :
+                                          &device->platformdev->dev;
+}
+
+resource_size_t
+nv_device_resource_start(struct nouveau_device *device, unsigned int bar);
+
+resource_size_t
+nv_device_resource_len(struct nouveau_device *device, unsigned int bar);
+
+dma_addr_t
+nv_device_map_page(struct nouveau_device *device, struct page *page);
+
+void
+nv_device_unmap_page(struct nouveau_device *device, dma_addr_t addr);
+
+int
+nv_device_get_irq(struct nouveau_device *device, bool stall);
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/core/namedb.h b/drivers/gpu/drm/nouveau/core/include/core/namedb.h
index 8897e0886085..f5b5fd8e1fc9 100644
--- a/drivers/gpu/drm/nouveau/core/include/core/namedb.h
+++ b/drivers/gpu/drm/nouveau/core/include/core/namedb.h
@@ -33,7 +33,7 @@ nv_namedb(void *obj)
 
 int  nouveau_namedb_create_(struct nouveau_object *, struct nouveau_object *,
                             struct nouveau_oclass *, u32 pclass,
-                            struct nouveau_oclass *, u32 engcls,
+                            struct nouveau_oclass *, u64 engcls,
                             int size, void **);
 
 int  _nouveau_namedb_ctor(struct nouveau_object *, struct nouveau_object *,
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/device.h b/drivers/gpu/drm/nouveau/core/include/engine/device.h
index b3dd2c4c2f1e..672d3c8f4145 100644
--- a/drivers/gpu/drm/nouveau/core/include/engine/device.h
+++ b/drivers/gpu/drm/nouveau/core/include/engine/device.h
@@ -3,11 +3,20 @@
 
 #include <core/device.h>
 
-#define nouveau_device_create(p,n,s,c,d,u)                                     \
-        nouveau_device_create_((p), (n), (s), (c), (d), sizeof(**u), (void **)u)
+struct platform_device;
 
-int  nouveau_device_create_(struct pci_dev *, u64 name, const char *sname,
-                            const char *cfg, const char *dbg, int, void **);
+enum nv_bus_type {
+        NOUVEAU_BUS_PCI,
+        NOUVEAU_BUS_PLATFORM,
+};
+
+#define nouveau_device_create(p,t,n,s,c,d,u)                                   \
+        nouveau_device_create_((void *)(p), (t), (n), (s), (c), (d),           \
+                               sizeof(**u), (void **)u)
+
+int  nouveau_device_create_(void *, enum nv_bus_type type, u64 name,
+                            const char *sname, const char *cfg, const char *dbg,
+                            int, void **);
 
 int nv04_identify(struct nouveau_device *);
 int nv10_identify(struct nouveau_device *);
@@ -17,6 +26,7 @@ int nv40_identify(struct nouveau_device *);
 int nv50_identify(struct nouveau_device *);
 int nvc0_identify(struct nouveau_device *);
 int nve0_identify(struct nouveau_device *);
+int gm100_identify(struct nouveau_device *);
 
 struct nouveau_device *nouveau_device_find(u64 name);
 
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/disp.h b/drivers/gpu/drm/nouveau/core/include/engine/disp.h
index 4b21fabfbddb..fd0c68804de3 100644
--- a/drivers/gpu/drm/nouveau/core/include/engine/disp.h
+++ b/drivers/gpu/drm/nouveau/core/include/engine/disp.h
@@ -36,14 +36,15 @@ void _nouveau_disp_dtor(struct nouveau_object *);
 #define _nouveau_disp_init _nouveau_engine_init
 #define _nouveau_disp_fini _nouveau_engine_fini
 
-extern struct nouveau_oclass nv04_disp_oclass;
-extern struct nouveau_oclass nv50_disp_oclass;
-extern struct nouveau_oclass nv84_disp_oclass;
-extern struct nouveau_oclass nva0_disp_oclass;
-extern struct nouveau_oclass nv94_disp_oclass;
-extern struct nouveau_oclass nva3_disp_oclass;
-extern struct nouveau_oclass nvd0_disp_oclass;
-extern struct nouveau_oclass nve0_disp_oclass;
-extern struct nouveau_oclass nvf0_disp_oclass;
+extern struct nouveau_oclass *nv04_disp_oclass;
+extern struct nouveau_oclass *nv50_disp_oclass;
+extern struct nouveau_oclass *nv84_disp_oclass;
+extern struct nouveau_oclass *nva0_disp_oclass;
+extern struct nouveau_oclass *nv94_disp_oclass;
+extern struct nouveau_oclass *nva3_disp_oclass;
+extern struct nouveau_oclass *nvd0_disp_oclass;
+extern struct nouveau_oclass *nve0_disp_oclass;
+extern struct nouveau_oclass *nvf0_disp_oclass;
+extern struct nouveau_oclass *gm107_disp_oclass;
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/engine/graph.h b/drivers/gpu/drm/nouveau/core/include/engine/graph.h
index 97705618de97..871edfdf3d5b 100644
--- a/drivers/gpu/drm/nouveau/core/include/engine/graph.h
+++ b/drivers/gpu/drm/nouveau/core/include/engine/graph.h
@@ -63,13 +63,14 @@ extern struct nouveau_oclass nv40_graph_oclass;
 extern struct nouveau_oclass nv50_graph_oclass;
 extern struct nouveau_oclass *nvc0_graph_oclass;
 extern struct nouveau_oclass *nvc1_graph_oclass;
-extern struct nouveau_oclass *nvc3_graph_oclass;
+extern struct nouveau_oclass *nvc4_graph_oclass;
 extern struct nouveau_oclass *nvc8_graph_oclass;
 extern struct nouveau_oclass *nvd7_graph_oclass;
 extern struct nouveau_oclass *nvd9_graph_oclass;
 extern struct nouveau_oclass *nve4_graph_oclass;
 extern struct nouveau_oclass *nvf0_graph_oclass;
 extern struct nouveau_oclass *nv108_graph_oclass;
+extern struct nouveau_oclass *gm107_graph_oclass;
 
 extern const struct nouveau_bitfield nv04_graph_nsource[];
 extern struct nouveau_ofuncs nv04_graph_ofuncs;
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/P0260.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/P0260.h
new file mode 100644
index 000000000000..bba01ab1e049
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/P0260.h
@@ -0,0 +1,23 @@
+#ifndef __NVBIOS_P0260_H__
+#define __NVBIOS_P0260_H__
+
+u32 nvbios_P0260Te(struct nouveau_bios *,
+                   u8 *ver, u8 *hdr, u8 *cnt, u8 *len, u8 *xnr, u8 *xsz);
+
+struct nvbios_P0260E {
+        u32 data;
+};
+
+u32 nvbios_P0260Ee(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr);
+u32 nvbios_P0260Ep(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr,
+                   struct nvbios_P0260E *);
+
+struct nvbios_P0260X {
+        u32 data;
+};
+
+u32 nvbios_P0260Xe(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr);
+u32 nvbios_P0260Xp(struct nouveau_bios *, int idx, u8 *ver, u8 *hdr,
+                   struct nvbios_P0260X *);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/conn.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/conn.h
index c1270548fd0d..a32feb3f3fb6 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/bios/conn.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/conn.h
@@ -16,6 +16,7 @@ enum dcb_connector_type {
         DCB_CONNECTOR_eDP = 0x47,
         DCB_CONNECTOR_HDMI_0 = 0x60,
         DCB_CONNECTOR_HDMI_1 = 0x61,
+        DCB_CONNECTOR_HDMI_C = 0x63,
         DCB_CONNECTOR_DMS59_DP0 = 0x64,
         DCB_CONNECTOR_DMS59_DP1 = 0x65,
         DCB_CONNECTOR_NONE = 0xff
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/ramcfg.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/ramcfg.h
index c5e6d1e6ac1d..c086ac6d677d 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/bios/ramcfg.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/ramcfg.h
@@ -61,6 +61,6 @@ struct nvbios_ramcfg {
 };
 
 u8 nvbios_ramcfg_count(struct nouveau_bios *);
-u8 nvbios_ramcfg_index(struct nouveau_bios *);
+u8 nvbios_ramcfg_index(struct nouveau_subdev *);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/bios/therm.h b/drivers/gpu/drm/nouveau/core/include/subdev/bios/therm.h
index 083541dbe9c8..8dc5051df55d 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/bios/therm.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/bios/therm.h
@@ -31,6 +31,12 @@ struct nouveau_therm_trip_point {
         int hysteresis;
 };
 
+enum nvbios_therm_fan_mode {
+        NVBIOS_THERM_FAN_TRIP = 0,
+        NVBIOS_THERM_FAN_LINEAR = 1,
+        NVBIOS_THERM_FAN_OTHER = 2,
+};
+
 struct nvbios_therm_fan {
         u16 pwm_freq;
 
@@ -40,6 +46,7 @@ struct nvbios_therm_fan {
         u16 bump_period;
         u16 slow_down_period;
 
+        enum nvbios_therm_fan_mode fan_mode;
         struct nouveau_therm_trip_point trip[NOUVEAU_TEMP_FAN_TRIP_MAX];
         u8 nr_fan_trip;
         u8 linear_min_temp;
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h b/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h
index ed1ac68c38b3..e292271a84e4 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/devinit.h
@@ -9,6 +9,7 @@ struct nouveau_devinit {
         bool post;
         void (*meminit)(struct nouveau_devinit *);
         int  (*pll_set)(struct nouveau_devinit *, u32 type, u32 freq);
+        u32  (*mmio)(struct nouveau_devinit *, u32 addr);
 };
 
 static inline struct nouveau_devinit *
@@ -28,5 +29,6 @@ extern struct nouveau_oclass *nv98_devinit_oclass;
 extern struct nouveau_oclass *nva3_devinit_oclass;
 extern struct nouveau_oclass *nvaf_devinit_oclass;
 extern struct nouveau_oclass *nvc0_devinit_oclass;
+extern struct nouveau_oclass *gm107_devinit_oclass;
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/fb.h b/drivers/gpu/drm/nouveau/core/include/subdev/fb.h
index d7ecafbae1ca..58c7ccdebb01 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/fb.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/fb.h
@@ -105,6 +105,7 @@ extern struct nouveau_oclass *nvaa_fb_oclass;
 extern struct nouveau_oclass *nvaf_fb_oclass;
 extern struct nouveau_oclass *nvc0_fb_oclass;
 extern struct nouveau_oclass *nve0_fb_oclass;
+extern struct nouveau_oclass *gm107_fb_oclass;
 
 #include <subdev/bios/ramcfg.h>
 
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/ltcg.h b/drivers/gpu/drm/nouveau/core/include/subdev/ltcg.h
index a1985ed3d58d..c9c1950b7743 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/ltcg.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/ltcg.h
@@ -35,6 +35,7 @@ nouveau_ltcg(void *obj)
 #define _nouveau_ltcg_init _nouveau_subdev_init
 #define _nouveau_ltcg_fini _nouveau_subdev_fini
 
-extern struct nouveau_oclass nvc0_ltcg_oclass;
+extern struct nouveau_oclass *gf100_ltcg_oclass;
+extern struct nouveau_oclass *gm107_ltcg_oclass;
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/mc.h b/drivers/gpu/drm/nouveau/core/include/subdev/mc.h
index 3c6738edd127..72b176831be6 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/mc.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/mc.h
@@ -12,6 +12,7 @@ struct nouveau_mc_intr {
 struct nouveau_mc {
         struct nouveau_subdev base;
         bool use_msi;
+        unsigned int irq;
 };
 
 static inline struct nouveau_mc *
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/therm.h b/drivers/gpu/drm/nouveau/core/include/subdev/therm.h
index 69891d4a3fe7..d4a68179e586 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/therm.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/therm.h
@@ -31,7 +31,7 @@ struct nouveau_therm {
         int (*pwm_ctrl)(struct nouveau_therm *, int line, bool);
         int (*pwm_get)(struct nouveau_therm *, int line, u32 *, u32 *);
         int (*pwm_set)(struct nouveau_therm *, int line, u32, u32);
-        int (*pwm_clock)(struct nouveau_therm *);
+        int (*pwm_clock)(struct nouveau_therm *, int line);
 
         int (*fan_get)(struct nouveau_therm *);
         int (*fan_set)(struct nouveau_therm *, int);
diff --git a/drivers/gpu/drm/nouveau/core/include/subdev/timer.h b/drivers/gpu/drm/nouveau/core/include/subdev/timer.h
index 9ab70dfe5b02..db9be803a874 100644
--- a/drivers/gpu/drm/nouveau/core/include/subdev/timer.h
+++ b/drivers/gpu/drm/nouveau/core/include/subdev/timer.h
@@ -59,5 +59,6 @@ int nouveau_timer_create_(struct nouveau_object *, struct nouveau_engine *,
                           struct nouveau_oclass *, int size, void **);
 
 extern struct nouveau_oclass nv04_timer_oclass;
+extern struct nouveau_oclass gk20a_timer_oclass;
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/os.h b/drivers/gpu/drm/nouveau/core/os.h
index 191e739f30d1..d0ced94ca54c 100644
--- a/drivers/gpu/drm/nouveau/core/os.h
+++ b/drivers/gpu/drm/nouveau/core/os.h
@@ -5,6 +5,7 @@
 #include <linux/slab.h>
 #include <linux/mutex.h>
 #include <linux/pci.h>
+#include <linux/platform_device.h>
 #include <linux/printk.h>
 #include <linux/bitops.h>
 #include <linux/firmware.h>
@@ -23,17 +24,6 @@
 
 #include <asm/unaligned.h>
 
-static inline int
-ffsll(u64 mask)
-{
-        int i;
-        for (i = 0; i < 64; i++) {
-                if (mask & (1ULL << i))
-                        return i + 1;
-        }
-        return 0;
-}
-
 #ifndef ioread32_native
 #ifdef __BIG_ENDIAN
 #define ioread16_native ioread16be
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bar/base.c b/drivers/gpu/drm/nouveau/core/subdev/bar/base.c
index 7098ddd54678..bdf594116f3f 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bar/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bar/base.c
@@ -118,8 +118,8 @@ nouveau_bar_create_(struct nouveau_object *parent,
         if (ret)
                 return ret;
 
-        bar->iomem = ioremap(pci_resource_start(device->pdev, 3),
-                             pci_resource_len(device->pdev, 3));
+        bar->iomem = ioremap(nv_device_resource_start(device, 3),
+                             nv_device_resource_len(device, 3));
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bar/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/bar/nv50.c
index 090d594a21b3..f748ba49dfc8 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bar/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bar/nv50.c
@@ -139,7 +139,7 @@ nv50_bar_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 
         /* BAR3 */
         start = 0x0100000000ULL;
-        limit = start + pci_resource_len(device->pdev, 3);
+        limit = start + nv_device_resource_len(device, 3);
 
         ret = nouveau_vm_new(device, start, limit, start, &vm);
         if (ret)
@@ -173,7 +173,7 @@ nv50_bar_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 
         /* BAR1 */
         start = 0x0000000000ULL;
-        limit = start + pci_resource_len(device->pdev, 1);
+        limit = start + nv_device_resource_len(device, 1);
 
         ret = nouveau_vm_new(device, start, limit--, start, &vm);
         if (ret)
@@ -231,7 +231,7 @@ static int
 nv50_bar_init(struct nouveau_object *object)
 {
         struct nv50_bar_priv *priv = (void *)object;
-        int ret;
+        int ret, i;
 
         ret = nouveau_bar_init(&priv->base);
         if (ret)
@@ -249,6 +249,8 @@ nv50_bar_init(struct nouveau_object *object)
         nv_wr32(priv, 0x001704, 0x40000000 | priv->mem->addr >> 12);
         nv_wr32(priv, 0x001708, 0x80000000 | priv->bar1->node->offset >> 4);
         nv_wr32(priv, 0x00170c, 0x80000000 | priv->bar3->node->offset >> 4);
+        for (i = 0; i < 8; i++)
+                nv_wr32(priv, 0x001900 + (i * 4), 0x00000000);
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bar/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/bar/nvc0.c
index bac5e754de35..3f30db62e656 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bar/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bar/nvc0.c
@@ -84,7 +84,6 @@ nvc0_bar_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
               struct nouveau_object **pobject)
 {
         struct nouveau_device *device = nv_device(parent);
-        struct pci_dev *pdev = device->pdev;
         struct nvc0_bar_priv *priv;
         struct nouveau_gpuobj *mem;
         struct nouveau_vm *vm;
@@ -107,14 +106,14 @@ nvc0_bar_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        ret = nouveau_vm_new(device, 0, pci_resource_len(pdev, 3), 0, &vm);
+        ret = nouveau_vm_new(device, 0, nv_device_resource_len(device, 3), 0, &vm);
         if (ret)
                 return ret;
 
         atomic_inc(&vm->engref[NVDEV_SUBDEV_BAR]);
 
         ret = nouveau_gpuobj_new(nv_object(priv), NULL,
-                                 (pci_resource_len(pdev, 3) >> 12) * 8,
+                                 (nv_device_resource_len(device, 3) >> 12) * 8,
                                  0x1000, NVOBJ_FLAG_ZERO_ALLOC,
                                  &vm->pgt[0].obj[0]);
         vm->pgt[0].refcount[0] = 1;
@@ -128,8 +127,8 @@ nvc0_bar_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 
         nv_wo32(mem, 0x0200, lower_32_bits(priv->bar[0].pgd->addr));
         nv_wo32(mem, 0x0204, upper_32_bits(priv->bar[0].pgd->addr));
-        nv_wo32(mem, 0x0208, lower_32_bits(pci_resource_len(pdev, 3) - 1));
-        nv_wo32(mem, 0x020c, upper_32_bits(pci_resource_len(pdev, 3) - 1));
+        nv_wo32(mem, 0x0208, lower_32_bits(nv_device_resource_len(device, 3) - 1));
+        nv_wo32(mem, 0x020c, upper_32_bits(nv_device_resource_len(device, 3) - 1));
 
         /* BAR1 */
         ret = nouveau_gpuobj_new(nv_object(priv), NULL, 0x1000, 0, 0,
@@ -143,7 +142,7 @@ nvc0_bar_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        ret = nouveau_vm_new(device, 0, pci_resource_len(pdev, 1), 0, &vm);
+        ret = nouveau_vm_new(device, 0, nv_device_resource_len(device, 1), 0, &vm);
         if (ret)
                 return ret;
 
@@ -156,8 +155,8 @@ nvc0_bar_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 
         nv_wo32(mem, 0x0200, lower_32_bits(priv->bar[1].pgd->addr));
         nv_wo32(mem, 0x0204, upper_32_bits(priv->bar[1].pgd->addr));
-        nv_wo32(mem, 0x0208, lower_32_bits(pci_resource_len(pdev, 1) - 1));
-        nv_wo32(mem, 0x020c, upper_32_bits(pci_resource_len(pdev, 1) - 1));
+        nv_wo32(mem, 0x0208, lower_32_bits(nv_device_resource_len(device, 1) - 1));
+        nv_wo32(mem, 0x020c, upper_32_bits(nv_device_resource_len(device, 1) - 1));
 
         priv->base.alloc = nouveau_bar_alloc;
         priv->base.kmap = nvc0_bar_kmap;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/P0260.c b/drivers/gpu/drm/nouveau/core/subdev/bios/P0260.c
new file mode 100644
index 000000000000..199f4e5f7488
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/P0260.c
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/bios.h>
+#include <subdev/bios/bit.h>
+#include <subdev/bios/ramcfg.h>
+#include <subdev/bios/P0260.h>
+
+u32
+nvbios_P0260Te(struct nouveau_bios *bios,
+               u8 *ver, u8 *hdr, u8 *cnt, u8 *len, u8 *xnr, u8 *xsz)
+{
+        struct bit_entry bit_P;
+        u32 data = 0x00000000;
+
+        if (!bit_entry(bios, 'P', &bit_P)) {
+                if (bit_P.version == 2 && bit_P.length > 0x63)
+                        data = nv_ro32(bios, bit_P.offset + 0x60);
+                if (data) {
+                        *ver = nv_ro08(bios, data + 0);
+                        switch (*ver) {
+                        case 0x10:
+                                *hdr = nv_ro08(bios, data + 1);
+                                *cnt = nv_ro08(bios, data + 2);
+                                *len = 4;
+                                *xnr = nv_ro08(bios, data + 3);
+                                *xsz = 4;
+                                return data;
+                        default:
+                                break;
+                        }
+                }
+        }
+
+        return 0x00000000;
+}
+
+u32
+nvbios_P0260Ee(struct nouveau_bios *bios, int idx, u8 *ver, u8 *len)
+{
+        u8  hdr, cnt, xnr, xsz;
+        u32 data = nvbios_P0260Te(bios, ver, &hdr, &cnt, len, &xnr, &xsz);
+        if (data && idx < cnt)
+                return data + hdr + (idx * *len);
+        return 0x00000000;
+}
+
+u32
+nvbios_P0260Ep(struct nouveau_bios *bios, int idx, u8 *ver, u8 *len,
+               struct nvbios_P0260E *info)
+{
+        u32 data = nvbios_P0260Ee(bios, idx, ver, len);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!data * *ver) {
+        case 0x10:
+                info->data = nv_ro32(bios, data);
+                return data;
+        default:
+                break;
+        }
+        return 0x00000000;
+}
+
+u32
+nvbios_P0260Xe(struct nouveau_bios *bios, int idx, u8 *ver, u8 *xsz)
+{
+        u8  hdr, cnt, len, xnr;
+        u32 data = nvbios_P0260Te(bios, ver, &hdr, &cnt, &len, &xnr, xsz);
+        if (data && idx < xnr)
+                return data + hdr + (cnt * len) + (idx * *xsz);
+        return 0x00000000;
+}
+
+u32
+nvbios_P0260Xp(struct nouveau_bios *bios, int idx, u8 *ver, u8 *hdr,
+               struct nvbios_P0260X *info)
+{
+        u32 data = nvbios_P0260Xe(bios, idx, ver, hdr);
+        memset(info, 0x00, sizeof(*info));
+        switch (!!data * *ver) {
+        case 0x10:
+                info->data = nv_ro32(bios, data);
+                return data;
+        default:
+                break;
+        }
+        return 0x00000000;
+}
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/base.c b/drivers/gpu/drm/nouveau/core/subdev/bios/base.c
index ef0c9c4a8cc3..e9df94f96d78 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/base.c
@@ -90,10 +90,26 @@ nouveau_bios_shadow_pramin(struct nouveau_bios *bios)
         int i;
 
         if (device->card_type >= NV_50) {
-                if (  device->card_type < NV_C0 ||
-                    !(nv_rd32(bios, 0x022500) & 0x00000001))
-                        addr = (u64)(nv_rd32(bios, 0x619f04) & 0xffffff00) << 8;
+                if (device->card_type >= NV_C0 && device->card_type < GM100) {
+                        if (nv_rd32(bios, 0x022500) & 0x00000001)
+                                return;
+                } else
+                if (device->card_type >= GM100) {
+                        if (nv_rd32(bios, 0x021c04) & 0x00000001)
+                                return;
+                }
+
+                addr = nv_rd32(bios, 0x619f04);
+                if (!(addr & 0x00000008)) {
+                        nv_debug(bios, "... not enabled\n");
+                        return;
+                }
+                if ( (addr & 0x00000003) != 1) {
+                        nv_debug(bios, "... not in vram\n");
+                        return;
+                }
 
+                addr = (u64)(addr >> 8) << 8;
                 if (!addr) {
                         addr  = (u64)nv_rd32(bios, 0x001700) << 16;
                         addr += 0xf0000;
@@ -141,6 +157,10 @@ nouveau_bios_shadow_prom(struct nouveau_bios *bios)
                 pcireg = 0x001850;
         access = nv_mask(bios, pcireg, 0x00000001, 0x00000000);
 
+        /* WARNING: PROM accesses should always be 32-bits aligned. Other
+         * accesses work on most chipset but do not on Kepler chipsets
+         */
+
         /* bail if no rom signature, with a workaround for a PROM reading
          * issue on some chipsets.  the first read after a period of
          * inactivity returns the wrong result, so retry the first header
@@ -148,31 +168,32 @@ nouveau_bios_shadow_prom(struct nouveau_bios *bios)
          */
         i = 16;
         do {
-                if (nv_rd08(bios, 0x300000) == 0x55)
+                if ((nv_rd32(bios, 0x300000) & 0xffff) == 0xaa55)
                         break;
         } while (i--);
 
-        if (!i || nv_rd08(bios, 0x300001) != 0xaa)
-                goto out;
-
-        /* additional check (see note below) - read PCI record header */
-        pcir = nv_rd08(bios, 0x300018) |
-               nv_rd08(bios, 0x300019) << 8;
-        if (nv_rd08(bios, 0x300000 + pcir) != 'P' ||
-            nv_rd08(bios, 0x300001 + pcir) != 'C' ||
-            nv_rd08(bios, 0x300002 + pcir) != 'I' ||
-            nv_rd08(bios, 0x300003 + pcir) != 'R')
+        if (!i)
                 goto out;
 
         /* read entire bios image to system memory */
-        bios->size = nv_rd08(bios, 0x300002) * 512;
+        bios->size = ((nv_rd32(bios, 0x300000) >> 16) & 0xff) * 512;
         if (!bios->size)
                 goto out;
 
         bios->data = kmalloc(bios->size, GFP_KERNEL);
         if (bios->data) {
-                for (i = 0; i < bios->size; i++)
-                        nv_wo08(bios, i, nv_rd08(bios, 0x300000 + i));
+                for (i = 0; i < bios->size; i+=4)
+                        nv_wo32(bios, i, nv_rd32(bios, 0x300000 + i));
+        }
+
+        /* check the PCI record header */
+        pcir = nv_ro16(bios, 0x0018);
+        if (bios->data[pcir + 0] != 'P' ||
+            bios->data[pcir + 1] != 'C' ||
+            bios->data[pcir + 2] != 'I' ||
+            bios->data[pcir + 3] != 'R') {
+                bios->size = 0;
+                kfree(bios->data);
         }
 
 out:
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c b/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c
index 2d9b9d7a7992..88606bfaf847 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/dcb.c
@@ -142,9 +142,36 @@ dcb_outp_parse(struct nouveau_bios *bios, u8 idx, u8 *ver, u8 *len,
                 if (*ver >= 0x40) {
                         u32 conf = nv_ro32(bios, dcb + 0x04);
                         switch (outp->type) {
+                        case DCB_OUTPUT_DP:
+                                switch (conf & 0x00e00000) {
+                                case 0x00000000:
+                                        outp->dpconf.link_bw = 0x06;
+                                        break;
+                                case 0x00200000:
+                                        outp->dpconf.link_bw = 0x0a;
+                                        break;
+                                case 0x00400000:
+                                default:
+                                        outp->dpconf.link_bw = 0x14;
+                                        break;
+                                }
+
+                                switch (conf & 0x0f000000) {
+                                case 0x0f000000:
+                                        outp->dpconf.link_nr = 4;
+                                        break;
+                                case 0x03000000:
+                                        outp->dpconf.link_nr = 2;
+                                        break;
+                                case 0x01000000:
+                                default:
+                                        outp->dpconf.link_nr = 1;
+                                        break;
+                                }
+
+                                /* fall-through... */
                         case DCB_OUTPUT_TMDS:
                         case DCB_OUTPUT_LVDS:
-                        case DCB_OUTPUT_DP:
                                 outp->link = (conf & 0x00000030) >> 4;
                                 outp->sorconf.link = outp->link; /*XXX*/
                                 outp->extdev = 0x00;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/init.c b/drivers/gpu/drm/nouveau/core/subdev/bios/init.c
index de201baeb053..acaeaf79e3f0 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/init.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/init.c
@@ -118,6 +118,8 @@ init_conn(struct nvbios_init *init)
 static inline u32
 init_nvreg(struct nvbios_init *init, u32 reg)
 {
+        struct nouveau_devinit *devinit = nouveau_devinit(init->bios);
+
         /* C51 (at least) sometimes has the lower bits set which the VBIOS
          * interprets to mean that access needs to go through certain IO
          * ports instead.  The NVIDIA binary driver has been seen to access
@@ -147,6 +149,9 @@ init_nvreg(struct nvbios_init *init, u32 reg)
 
         if (reg & ~0x00fffffc)
                 warn("unknown bits in register 0x%08x\n", reg);
+
+        if (devinit->mmio)
+                reg = devinit->mmio(devinit, reg);
         return reg;
 }
 
@@ -154,7 +159,7 @@ static u32
 init_rd32(struct nvbios_init *init, u32 reg)
 {
         reg = init_nvreg(init, reg);
-        if (init_exec(init))
+        if (reg != ~0 && init_exec(init))
                 return nv_rd32(init->subdev, reg);
         return 0x00000000;
 }
@@ -163,7 +168,7 @@ static void
 init_wr32(struct nvbios_init *init, u32 reg, u32 val)
 {
         reg = init_nvreg(init, reg);
-        if (init_exec(init))
+        if (reg != ~0 && init_exec(init))
                 nv_wr32(init->subdev, reg, val);
 }
 
@@ -171,7 +176,7 @@ static u32
 init_mask(struct nvbios_init *init, u32 reg, u32 mask, u32 val)
 {
         reg = init_nvreg(init, reg);
-        if (init_exec(init)) {
+        if (reg != ~0 && init_exec(init)) {
                 u32 tmp = nv_rd32(init->subdev, reg);
                 nv_wr32(init->subdev, reg, (tmp & ~mask) | val);
                 return tmp;
@@ -410,7 +415,7 @@ init_ram_restrict(struct nvbios_init *init)
          * in case *not* re-reading the strap causes similar breakage.
          */
         if (!init->ramcfg || init->bios->version.major < 0x70)
-                init->ramcfg = 0x80000000 | nvbios_ramcfg_index(init->bios);
+                init->ramcfg = 0x80000000 | nvbios_ramcfg_index(init->subdev);
         return (init->ramcfg & 0x7fffffff);
 }
 
@@ -845,9 +850,8 @@ init_idx_addr_latched(struct nvbios_init *init)
         u32 data = nv_ro32(bios, init->offset + 13);
         u8 count = nv_ro08(bios, init->offset + 17);
 
-        trace("INDEX_ADDRESS_LATCHED\t"
-              "R[0x%06x] : R[0x%06x]\n\tCTRL &= 0x%08x |= 0x%08x\n",
-              creg, dreg, mask, data);
+        trace("INDEX_ADDRESS_LATCHED\tR[0x%06x] : R[0x%06x]\n", creg, dreg);
+        trace("\tCTRL &= 0x%08x |= 0x%08x\n", mask, data);
         init->offset += 18;
 
         while (count--) {
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/ramcfg.c b/drivers/gpu/drm/nouveau/core/subdev/bios/ramcfg.c
index 991aedda999b..6c401f70ab99 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/ramcfg.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/ramcfg.c
@@ -27,9 +27,9 @@
 #include <subdev/bios/ramcfg.h>
 
 static u8
-nvbios_ramcfg_strap(struct nouveau_bios *bios)
+nvbios_ramcfg_strap(struct nouveau_subdev *subdev)
 {
-        return (nv_rd32(bios, 0x101000) & 0x0000003c) >> 2;
+        return (nv_rd32(subdev, 0x101000) & 0x0000003c) >> 2;
 }
 
 u8
@@ -48,9 +48,10 @@ nvbios_ramcfg_count(struct nouveau_bios *bios)
 }
 
 u8
-nvbios_ramcfg_index(struct nouveau_bios *bios)
+nvbios_ramcfg_index(struct nouveau_subdev *subdev)
 {
-        u8 strap = nvbios_ramcfg_strap(bios);
+        struct nouveau_bios *bios = nouveau_bios(subdev);
+        u8 strap = nvbios_ramcfg_strap(subdev);
         u32 xlat = 0x00000000;
         struct bit_entry bit_M;
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/bios/therm.c b/drivers/gpu/drm/nouveau/core/subdev/bios/therm.c
index 22ac6dbd6c8f..d15854094078 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/bios/therm.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/bios/therm.c
@@ -164,6 +164,7 @@ nvbios_therm_fan_parse(struct nouveau_bios *bios,
 
         i = 0;
         fan->nr_fan_trip = 0;
+        fan->fan_mode = NVBIOS_THERM_FAN_OTHER;
         while ((entry = nvbios_therm_entry(bios, i++, &ver, &len))) {
                 s16 value = nv_ro16(bios, entry + 1);
 
@@ -174,6 +175,8 @@ nvbios_therm_fan_parse(struct nouveau_bios *bios,
                         break;
                 case 0x24:
                         fan->nr_fan_trip++;
+                        if (fan->fan_mode > NVBIOS_THERM_FAN_TRIP)
+                                fan->fan_mode = NVBIOS_THERM_FAN_TRIP;
                         cur_trip = &fan->trip[fan->nr_fan_trip - 1];
                         cur_trip->hysteresis = value & 0xf;
                         cur_trip->temp = (value & 0xff0) >> 4;
@@ -194,11 +197,19 @@ nvbios_therm_fan_parse(struct nouveau_bios *bios,
                         fan->slow_down_period = value;
                         break;
                 case 0x46:
+                        if (fan->fan_mode > NVBIOS_THERM_FAN_LINEAR)
+                                fan->fan_mode = NVBIOS_THERM_FAN_LINEAR;
                         fan->linear_min_temp = nv_ro08(bios, entry + 1);
                         fan->linear_max_temp = nv_ro08(bios, entry + 2);
                         break;
                 }
         }
 
+        /* starting from fermi, fan management is always linear */
+        if (nv_device(bios)->card_type >= NV_C0 &&
+                fan->fan_mode == NVBIOS_THERM_FAN_OTHER) {
+                fan->fan_mode = NVBIOS_THERM_FAN_LINEAR;
+        }
+
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/base.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/base.c
index 8fa34e8152c2..239acfe876c3 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/base.c
@@ -96,5 +96,6 @@ nouveau_devinit_create_(struct nouveau_object *parent,
         devinit->post = nouveau_boolopt(device->cfgopt, "NvForcePost", false);
         devinit->meminit = impl->meminit;
         devinit->pll_set = impl->pll_set;
+        devinit->mmio    = impl->mmio;
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/fbmem.h b/drivers/gpu/drm/nouveau/core/subdev/devinit/fbmem.h
index 6b56a0f4cb40..4fe49cf4c99a 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/fbmem.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/fbmem.h
@@ -24,6 +24,8 @@
  *
  */
 
+#include <core/device.h>
+
 #define NV04_PFB_BOOT_0                                         0x00100000
 #       define NV04_PFB_BOOT_0_RAM_AMOUNT                       0x00000003
 #       define NV04_PFB_BOOT_0_RAM_AMOUNT_32MB                  0x00000000
@@ -60,10 +62,10 @@
 #       define NV10_PFB_REFCTRL_VALID_1                         (1 << 31)
 
 static inline struct io_mapping *
-fbmem_init(struct pci_dev *pdev)
+fbmem_init(struct nouveau_device *dev)
 {
-        return io_mapping_create_wc(pci_resource_start(pdev, 1),
-                                    pci_resource_len(pdev, 1));
+        return io_mapping_create_wc(nv_device_resource_start(dev, 1),
+                                    nv_device_resource_len(dev, 1));
 }
 
 static inline void
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/gm107.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/gm107.c
new file mode 100644
index 000000000000..c69bc7f54e37
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/gm107.c
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv50.h"
+
+static u64
+gm107_devinit_disable(struct nouveau_devinit *devinit)
+{
+        struct nv50_devinit_priv *priv = (void *)devinit;
+        u32 r021c00 = nv_rd32(priv, 0x021c00);
+        u32 r021c04 = nv_rd32(priv, 0x021c04);
+        u64 disable = 0ULL;
+
+        if (r021c00 & 0x00000001)
+                disable |= (1ULL << NVDEV_ENGINE_COPY0);
+        if (r021c00 & 0x00000004)
+                disable |= (1ULL << NVDEV_ENGINE_COPY2);
+        if (r021c04 & 0x00000001)
+                disable |= (1ULL << NVDEV_ENGINE_DISP);
+
+        return disable;
+}
+
+struct nouveau_oclass *
+gm107_devinit_oclass = &(struct nouveau_devinit_impl) {
+        .base.handle = NV_SUBDEV(DEVINIT, 0x07),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv50_devinit_ctor,
+                .dtor = _nouveau_devinit_dtor,
+                .init = nv50_devinit_init,
+                .fini = _nouveau_devinit_fini,
+        },
+        .pll_set = nvc0_devinit_pll_set,
+        .disable = gm107_devinit_disable,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c
index 7037eae46e44..052ad690b468 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv04.c
@@ -38,7 +38,7 @@ nv04_devinit_meminit(struct nouveau_devinit *devinit)
         int i;
 
         /* Map the framebuffer aperture */
-        fb = fbmem_init(nv_device(priv)->pdev);
+        fb = fbmem_init(nv_device(priv));
         if (!fb) {
                 nv_error(priv, "failed to map fb\n");
                 return;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c
index 98b7e6780dc7..4a19c10e5178 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv05.c
@@ -53,7 +53,7 @@ nv05_devinit_meminit(struct nouveau_devinit *devinit)
         int i, v;
 
         /* Map the framebuffer aperture */
-        fb = fbmem_init(nv_device(priv)->pdev);
+        fb = fbmem_init(nv_device(priv));
         if (!fb) {
                 nv_error(priv, "failed to map fb\n");
                 return;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c
index 32b3d2131a7f..3b8d657da279 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv10.c
@@ -46,7 +46,7 @@ nv10_devinit_meminit(struct nouveau_devinit *devinit)
                 mem_width_count = 2;
 
         /* Map the framebuffer aperture */
-        fb = fbmem_init(nv_device(priv)->pdev);
+        fb = fbmem_init(nv_device(priv));
         if (!fb) {
                 nv_error(priv, "failed to map fb\n");
                 return;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c
index 4689ba303b0b..04bc9732644c 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv20.c
@@ -37,7 +37,7 @@ nv20_devinit_meminit(struct nouveau_devinit *devinit)
         struct io_mapping *fb;
 
         /* Map the framebuffer aperture */
-        fb = fbmem_init(nv_device(priv)->pdev);
+        fb = fbmem_init(nv_device(priv));
         if (!fb) {
                 nv_error(priv, "failed to map fb\n");
                 return;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.h b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.h
index 141c27e9f182..51d5076333ec 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nv50.h
@@ -5,6 +5,7 @@
 
 struct nv50_devinit_priv {
         struct nouveau_devinit base;
+        u32 r001540;
 };
 
 int  nv50_devinit_ctor(struct nouveau_object *, struct nouveau_object *,
@@ -15,4 +16,6 @@ int  nv50_devinit_pll_set(struct nouveau_devinit *, u32, u32);
 
 int  nva3_devinit_pll_set(struct nouveau_devinit *, u32, u32);
 
+int  nvc0_devinit_pll_set(struct nouveau_devinit *, u32, u32);
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nva3.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nva3.c
index 6dedf1dad7f7..006cf348bda7 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nva3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nva3.c
@@ -81,6 +81,55 @@ nva3_devinit_disable(struct nouveau_devinit *devinit)
         return disable;
 }
 
+static u32
+nva3_devinit_mmio_part[] = {
+        0x100720, 0x1008bc, 4,
+        0x100a20, 0x100adc, 4,
+        0x100d80, 0x100ddc, 4,
+        0x110000, 0x110f9c, 4,
+        0x111000, 0x11103c, 8,
+        0x111080, 0x1110fc, 4,
+        0x111120, 0x1111fc, 4,
+        0x111300, 0x1114bc, 4,
+        0,
+};
+
+static u32
+nva3_devinit_mmio(struct nouveau_devinit *devinit, u32 addr)
+{
+        struct nv50_devinit_priv *priv = (void *)devinit;
+        u32 *mmio = nva3_devinit_mmio_part;
+
+        /* the init tables on some boards have INIT_RAM_RESTRICT_ZM_REG_GROUP
+         * instructions which touch registers that may not even exist on
+         * some configurations (Quadro 400), which causes the register
+         * interface to screw up for some amount of time after attempting to
+         * write to one of these, and results in all sorts of things going
+         * horribly wrong.
+         *
+         * the binary driver avoids touching these registers at all, however,
+         * the video bios doesn't care and does what the scripts say.  it's
+         * presumed that the io-port access to priv registers isn't effected
+         * by the screw-up bug mentioned above.
+         *
+         * really, a new opcode should've been invented to handle these
+         * requirements, but whatever, it's too late for that now.
+         */
+        while (mmio[0]) {
+                if (addr >= mmio[0] && addr <= mmio[1]) {
+                        u32 part = (addr / mmio[2]) & 7;
+                        if (!priv->r001540)
+                                priv->r001540 = nv_rd32(priv, 0x001540);
+                        if (part >= hweight8((priv->r001540 >> 16) & 0xff))
+                                return ~0;
+                        return addr;
+                }
+                mmio += 3;
+        }
+
+        return addr;
+}
+
 struct nouveau_oclass *
 nva3_devinit_oclass = &(struct nouveau_devinit_impl) {
         .base.handle = NV_SUBDEV(DEVINIT, 0xa3),
@@ -92,4 +141,5 @@ nva3_devinit_oclass = &(struct nouveau_devinit_impl) {
         },
         .pll_set = nva3_devinit_pll_set,
         .disable = nva3_devinit_disable,
+        .mmio    = nva3_devinit_mmio,
 }.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/devinit/nvc0.c
index fa7e63766b1b..30c765747eea 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/nvc0.c
@@ -24,7 +24,7 @@
 
 #include "nv50.h"
 
-static int
+int
 nvc0_devinit_pll_set(struct nouveau_devinit *devinit, u32 type, u32 freq)
 {
         struct nv50_devinit_priv *priv = (void *)devinit;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/devinit/priv.h b/drivers/gpu/drm/nouveau/core/subdev/devinit/priv.h
index 822a2fbf44a5..f0e8683ad840 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/devinit/priv.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/devinit/priv.h
@@ -11,6 +11,7 @@ struct nouveau_devinit_impl {
         void (*meminit)(struct nouveau_devinit *);
         int  (*pll_set)(struct nouveau_devinit *, u32 type, u32 freq);
         u64  (*disable)(struct nouveau_devinit *);
+        u32  (*mmio)(struct nouveau_devinit *, u32);
 };
 
 #define nouveau_devinit_create(p,e,o,d)                                        \
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/gm107.c b/drivers/gpu/drm/nouveau/core/subdev/fb/gm107.c
new file mode 100644
index 000000000000..c4840aedc2dc
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/gm107.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nvc0.h"
+
+struct nouveau_oclass *
+gm107_fb_oclass = &(struct nouveau_fb_impl) {
+        .base.handle = NV_SUBDEV(FB, 0x07),
+        .base.ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nvc0_fb_ctor,
+                .dtor = nvc0_fb_dtor,
+                .init = nvc0_fb_init,
+                .fini = _nouveau_fb_fini,
+        },
+        .memtype = nvc0_fb_memtype_valid,
+        .ram = &gm107_ram_oclass,
+}.base;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c
index cbc7f00c1278..1fc55c1e91a1 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nv50.c
@@ -250,10 +250,8 @@ nv50_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 
         priv->r100c08_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
         if (priv->r100c08_page) {
-                priv->r100c08 = pci_map_page(device->pdev, priv->r100c08_page,
-                                             0, PAGE_SIZE,
-                                             PCI_DMA_BIDIRECTIONAL);
-                if (pci_dma_mapping_error(device->pdev, priv->r100c08))
+                priv->r100c08 = nv_device_map_page(device, priv->r100c08_page);
+                if (!priv->r100c08)
                         nv_warn(priv, "failed 0x100c08 page map\n");
         } else {
                 nv_warn(priv, "failed 0x100c08 page alloc\n");
@@ -270,8 +268,7 @@ nv50_fb_dtor(struct nouveau_object *object)
         struct nv50_fb_priv *priv = (void *)object;
 
         if (priv->r100c08_page) {
-                pci_unmap_page(device->pdev, priv->r100c08, PAGE_SIZE,
-                               PCI_DMA_BIDIRECTIONAL);
+                nv_device_unmap_page(device, priv->r100c08);
                 __free_page(priv->r100c08_page);
         }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c
index 45470e1f0385..0670ae33ee45 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.c
@@ -70,8 +70,7 @@ nvc0_fb_dtor(struct nouveau_object *object)
         struct nvc0_fb_priv *priv = (void *)object;
 
         if (priv->r100c10_page) {
-                pci_unmap_page(device->pdev, priv->r100c10, PAGE_SIZE,
-                               PCI_DMA_BIDIRECTIONAL);
+                nv_device_unmap_page(device, priv->r100c10);
                 __free_page(priv->r100c10_page);
         }
 
@@ -94,10 +93,8 @@ nvc0_fb_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
 
         priv->r100c10_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
         if (priv->r100c10_page) {
-                priv->r100c10 = pci_map_page(device->pdev, priv->r100c10_page,
-                                             0, PAGE_SIZE,
-                                             PCI_DMA_BIDIRECTIONAL);
-                if (pci_dma_mapping_error(device->pdev, priv->r100c10))
+                priv->r100c10 = nv_device_map_page(device, priv->r100c10_page);
+                if (!priv->r100c10)
                         return -EFAULT;
         }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.h b/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.h
index 9e1931eb746f..705a06d755ad 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/nvc0.h
@@ -18,12 +18,14 @@ int  nvc0_fb_init(struct nouveau_object *);
 bool nvc0_fb_memtype_valid(struct nouveau_fb *, u32);
 
 
-#define nvc0_ram_create(p,e,o,d)                                               \
-        nvc0_ram_create_((p), (e), (o), sizeof(**d), (void **)d)
+#define nvc0_ram_create(p,e,o,m,d)                                             \
+        nvc0_ram_create_((p), (e), (o), (m), sizeof(**d), (void **)d)
 int  nvc0_ram_create_(struct nouveau_object *, struct nouveau_object *,
-                      struct nouveau_oclass *, int, void **);
+                      struct nouveau_oclass *, u32, int, void **);
 int  nvc0_ram_get(struct nouveau_fb *, u64, u32, u32, u32,
                   struct nouveau_mem **);
 void nvc0_ram_put(struct nouveau_fb *, struct nouveau_mem **);
 
+int  nve0_ram_init(struct nouveau_object*);
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h b/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h
index edaf95dee612..da74c889aed4 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/priv.h
@@ -32,6 +32,7 @@ extern struct nouveau_oclass nva3_ram_oclass;
 extern struct nouveau_oclass nvaa_ram_oclass;
 extern struct nouveau_oclass nvc0_ram_oclass;
 extern struct nouveau_oclass nve0_ram_oclass;
+extern struct nouveau_oclass gm107_ram_oclass;
 
 int nouveau_sddr3_calc(struct nouveau_ram *ram);
 int nouveau_gddr5_calc(struct nouveau_ram *ram, bool nuts);
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramgm107.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramgm107.c
new file mode 100644
index 000000000000..4c6363595c79
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramgm107.c
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nvc0.h"
+
+struct gm107_ram {
+        struct nouveau_ram base;
+};
+
+static int
+gm107_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+               struct nouveau_oclass *oclass, void *data, u32 size,
+               struct nouveau_object **pobject)
+{
+        struct gm107_ram *ram;
+        int ret;
+
+        ret = nvc0_ram_create(parent, engine, oclass, 0x021c14, &ram);
+        *pobject = nv_object(ram);
+        if (ret)
+                return ret;
+
+        return 0;
+}
+
+struct nouveau_oclass
+gm107_ram_oclass = {
+        .handle = 0,
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = gm107_ram_ctor,
+                .dtor = _nouveau_ram_dtor,
+                .init = nve0_ram_init,
+                .fini = _nouveau_ram_fini,
+        }
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv50.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv50.c
index c7fdb3a9e88b..ef91b6e893af 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnv50.c
@@ -91,7 +91,7 @@ nv50_ram_calc(struct nouveau_fb *pfb, u32 freq)
         } while (perfE.memory < freq);
 
         /* locate specific data set for the attached memory */
-        strap = nvbios_ramcfg_index(bios);
+        strap = nvbios_ramcfg_index(nv_subdev(pfb));
         if (strap >= cnt) {
                 nv_error(pfb, "invalid ramcfg strap\n");
                 return -EINVAL;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c
index f4ae8aa46a25..6eb97f16fbda 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnva3.c
@@ -98,7 +98,7 @@ nva3_ram_calc(struct nouveau_fb *pfb, u32 freq)
         }
 
         /* locate specific data set for the attached memory */
-        strap = nvbios_ramcfg_index(bios);
+        strap = nvbios_ramcfg_index(nv_subdev(pfb));
         if (strap >= cnt) {
                 nv_error(pfb, "invalid ramcfg strap\n");
                 return -EINVAL;
@@ -335,21 +335,23 @@ nva3_ram_init(struct nouveau_object *object)
         /* prepare for ddr link training, and load training patterns */
         switch (ram->base.type) {
         case NV_MEM_TYPE_DDR3: {
-                static const u32 pattern[16] = {
-                        0xaaaaaaaa, 0xcccccccc, 0xdddddddd, 0xeeeeeeee,
-                        0x00000000, 0x11111111, 0x44444444, 0xdddddddd,
-                        0x33333333, 0x55555555, 0x77777777, 0x66666666,
-                        0x99999999, 0x88888888, 0xeeeeeeee, 0xbbbbbbbb,
-                };
-
-                nv_wr32(pfb, 0x100538, 0x10001ff6); /*XXX*/
-                nv_wr32(pfb, 0x1005a8, 0x0000ffff);
-                nv_mask(pfb, 0x10f800, 0x00000001, 0x00000001);
-                for (i = 0; i < 0x30; i++) {
-                        nv_wr32(pfb, 0x10f8c0, (i << 8) | i);
-                        nv_wr32(pfb, 0x10f8e0, (i << 8) | i);
-                        nv_wr32(pfb, 0x10f900, pattern[i % 16]);
-                        nv_wr32(pfb, 0x10f920, pattern[i % 16]);
+                if (nv_device(pfb)->chipset == 0xa8) {
+                        static const u32 pattern[16] = {
+                                0xaaaaaaaa, 0xcccccccc, 0xdddddddd, 0xeeeeeeee,
+                                0x00000000, 0x11111111, 0x44444444, 0xdddddddd,
+                                0x33333333, 0x55555555, 0x77777777, 0x66666666,
+                                0x99999999, 0x88888888, 0xeeeeeeee, 0xbbbbbbbb,
+                        };
+
+                        nv_wr32(pfb, 0x100538, 0x10001ff6); /*XXX*/
+                        nv_wr32(pfb, 0x1005a8, 0x0000ffff);
+                        nv_mask(pfb, 0x10f800, 0x00000001, 0x00000001);
+                        for (i = 0; i < 0x30; i++) {
+                                nv_wr32(pfb, 0x10f8c0, (i << 8) | i);
+                                nv_wr32(pfb, 0x10f8e0, (i << 8) | i);
+                                nv_wr32(pfb, 0x10f900, pattern[i % 16]);
+                                nv_wr32(pfb, 0x10f920, pattern[i % 16]);
+                        }
                 }
         }
                 break;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvc0.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvc0.c
index 0391b824ee76..8edc92224c84 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnvc0.c
@@ -152,7 +152,7 @@ nvc0_ram_calc(struct nouveau_fb *pfb, u32 freq)
         }
 
         /* locate specific data set for the attached memory */
-        strap = nvbios_ramcfg_index(bios);
+        strap = nvbios_ramcfg_index(nv_subdev(pfb));
         if (strap >= cnt) {
                 nv_error(pfb, "invalid ramcfg strap\n");
                 return -EINVAL;
@@ -505,7 +505,8 @@ nvc0_ram_get(struct nouveau_fb *pfb, u64 size, u32 align, u32 ncmin,
 
 int
 nvc0_ram_create_(struct nouveau_object *parent, struct nouveau_object *engine,
-                 struct nouveau_oclass *oclass, int size, void **pobject)
+                 struct nouveau_oclass *oclass, u32 maskaddr, int size,
+                 void **pobject)
 {
         struct nouveau_fb *pfb = nouveau_fb(parent);
         struct nouveau_bios *bios = nouveau_bios(pfb);
@@ -513,7 +514,7 @@ nvc0_ram_create_(struct nouveau_object *parent, struct nouveau_object *engine,
         const u32 rsvd_head = ( 256 * 1024) >> 12; /* vga memory */
         const u32 rsvd_tail = (1024 * 1024) >> 12; /* vbios etc */
         u32 parts = nv_rd32(pfb, 0x022438);
-        u32 pmask = nv_rd32(pfb, 0x022554);
+        u32 pmask = nv_rd32(pfb, maskaddr);
         u32 bsize = nv_rd32(pfb, 0x10f20c);
         u32 offset, length;
         bool uniform = true;
@@ -630,7 +631,7 @@ nvc0_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         struct nvc0_ram *ram;
         int ret;
 
-        ret = nvc0_ram_create(parent, engine, oclass, &ram);
+        ret = nvc0_ram_create(parent, engine, oclass, 0x022554, &ram);
         *pobject = nv_object(ram);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnve0.c b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnve0.c
index 3257c522a021..16752192cf87 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/fb/ramnve0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/fb/ramnve0.c
@@ -950,10 +950,11 @@ nve0_ram_calc_data(struct nouveau_fb *pfb, u32 freq,
         }
 
         /* locate specific data set for the attached memory */
+        strap = nvbios_ramcfg_index(nv_subdev(pfb));
         ram->base.ramcfg.data = nvbios_rammapSp(bios, ram->base.rammap.data,
                                                 ram->base.rammap.version,
-                                                ram->base.rammap.size, cnt, len,
-                                                nvbios_ramcfg_index(bios),
+                                                ram->base.rammap.size,
+                                                cnt, len, strap,
                                                 &ram->base.ramcfg.version,
                                                 &ram->base.ramcfg.size,
                                                 &data->bios);
@@ -1123,7 +1124,7 @@ nve0_ram_tidy(struct nouveau_fb *pfb)
         ram_exec(fuc, false);
 }
 
-static int
+int
 nve0_ram_init(struct nouveau_object *object)
 {
         struct nouveau_fb *pfb = (void *)object->parent;
@@ -1226,7 +1227,7 @@ nve0_ram_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         int ret, i;
         u32 tmp;
 
-        ret = nvc0_ram_create(parent, engine, oclass, &ram);
+        ret = nvc0_ram_create(parent, engine, oclass, 0x022554, &ram);
         *pobject = nv_object(ram);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c
index c4c1d415e7fe..2ef774731629 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/gpio/nv50.c
@@ -46,7 +46,8 @@ nv50_gpio_reset(struct nouveau_gpio *gpio, u8 match)
                 u8  unk0 = !!(data & 0x02000000);
                 u8  unk1 = !!(data & 0x04000000);
                 u32 val = (unk1 << 16) | unk0;
-                u32 reg = regs[line >> 4]; line &= 0x0f;
+                u32 reg = regs[line >> 4];
+                u32 lsh = line & 0x0f;
 
                 if ( func  == DCB_GPIO_UNUSED ||
                     (match != DCB_GPIO_UNUSED && match != func))
@@ -54,7 +55,7 @@ nv50_gpio_reset(struct nouveau_gpio *gpio, u8 match)
 
                 gpio->set(gpio, 0, func, line, defs);
 
-                nv_mask(priv, reg, 0x00010001 << line, val << line);
+                nv_mask(priv, reg, 0x00010001 << lsh, val << lsh);
         }
 }
 
@@ -79,7 +80,7 @@ nv50_gpio_drive(struct nouveau_gpio *gpio, int line, int dir, int out)
         if (nv50_gpio_location(line, &reg, &shift))
                 return -EINVAL;
 
-        nv_mask(gpio, reg, 7 << shift, (((dir ^ 1) << 1) | out) << shift);
+        nv_mask(gpio, reg, 3 << shift, (((dir ^ 1) << 1) | out) << shift);
         return 0;
 }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c b/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
index c33c03d2f4af..378e05b88e6f 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/i2c/base.c
@@ -111,7 +111,7 @@ nouveau_i2c_port_create_(struct nouveau_object *parent,
         snprintf(port->adapter.name, sizeof(port->adapter.name),
                  "nouveau-%s-%d", device->name, index);
         port->adapter.owner = THIS_MODULE;
-        port->adapter.dev.parent = &device->pdev->dev;
+        port->adapter.dev.parent = nv_device_base(device);
         port->index = index;
         port->func = func;
         i2c_set_adapdata(&port->adapter, i2c);
diff --git a/drivers/gpu/drm/nouveau/core/subdev/instmem/nv40.c b/drivers/gpu/drm/nouveau/core/subdev/instmem/nv40.c
index ec0b9661d614..8803809f9fc5 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/instmem/nv40.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/instmem/nv40.c
@@ -50,7 +50,6 @@ nv40_instmem_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                   struct nouveau_object **pobject)
 {
         struct nouveau_device *device = nv_device(parent);
-        struct pci_dev *pdev = device->pdev;
         struct nv04_instmem_priv *priv;
         int ret, bar, vs;
 
@@ -60,13 +59,13 @@ nv40_instmem_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                 return ret;
 
         /* map bar */
-        if (pci_resource_len(pdev, 2))
+        if (nv_device_resource_len(device, 2))
                 bar = 2;
         else
                 bar = 3;
 
-        priv->iomem = ioremap(pci_resource_start(pdev, bar),
-                              pci_resource_len(pdev, bar));
+        priv->iomem = ioremap(nv_device_resource_start(device, bar),
+                              nv_device_resource_len(device, bar));
         if (!priv->iomem) {
                 nv_error(priv, "unable to map PRAMIN BAR\n");
                 return -EFAULT;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/ltcg/nvc0.c b/drivers/gpu/drm/nouveau/core/subdev/ltcg/gf100.c
index cce65cc56514..f2f3338a967a 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/ltcg/nvc0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/ltcg/gf100.c
@@ -22,44 +22,35 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/ltcg.h>
 #include <subdev/fb.h>
 #include <subdev/timer.h>
 
-struct nvc0_ltcg_priv {
-        struct nouveau_ltcg base;
-        u32 part_nr;
-        u32 subp_nr;
-        u32 num_tags;
-        u32 tag_base;
-        struct nouveau_mm tags;
-        struct nouveau_mm_node *tag_ram;
-};
+#include "gf100.h"
 
 static void
-nvc0_ltcg_subp_isr(struct nvc0_ltcg_priv *priv, int unit, int subp)
+gf100_ltcg_lts_isr(struct gf100_ltcg_priv *priv, int ltc, int lts)
 {
-        u32 subp_base = 0x141000 + (unit * 0x2000) + (subp * 0x400);
-        u32 stat = nv_rd32(priv, subp_base + 0x020);
+        u32 base = 0x141000 + (ltc * 0x2000) + (lts * 0x400);
+        u32 stat = nv_rd32(priv, base + 0x020);
 
         if (stat) {
-                nv_info(priv, "LTC%d_LTS%d: 0x%08x\n", unit, subp, stat);
-                nv_wr32(priv, subp_base + 0x020, stat);
+                nv_info(priv, "LTC%d_LTS%d: 0x%08x\n", ltc, lts, stat);
+                nv_wr32(priv, base + 0x020, stat);
         }
 }
 
 static void
-nvc0_ltcg_intr(struct nouveau_subdev *subdev)
+gf100_ltcg_intr(struct nouveau_subdev *subdev)
 {
-        struct nvc0_ltcg_priv *priv = (void *)subdev;
-        u32 units;
-
-        units = nv_rd32(priv, 0x00017c);
-        while (units) {
-                u32 subp, unit = ffs(units) - 1;
-                for (subp = 0; subp < priv->subp_nr; subp++)
-                        nvc0_ltcg_subp_isr(priv, unit, subp);
-                units &= ~(1 << unit);
+        struct gf100_ltcg_priv *priv = (void *)subdev;
+        u32 mask;
+
+        mask = nv_rd32(priv, 0x00017c);
+        while (mask) {
+                u32 lts, ltc = __ffs(mask);
+                for (lts = 0; lts < priv->lts_nr; lts++)
+                        gf100_ltcg_lts_isr(priv, ltc, lts);
+                mask &= ~(1 << ltc);
         }
 
         /* we do something horribly wrong and upset PMFB a lot, so mask off
@@ -68,11 +59,11 @@ nvc0_ltcg_intr(struct nouveau_subdev *subdev)
         nv_mask(priv, 0x000640, 0x02000000, 0x00000000);
 }
 
-static int
-nvc0_ltcg_tags_alloc(struct nouveau_ltcg *ltcg, u32 n,
+int
+gf100_ltcg_tags_alloc(struct nouveau_ltcg *ltcg, u32 n,
                      struct nouveau_mm_node **pnode)
 {
-        struct nvc0_ltcg_priv *priv = (struct nvc0_ltcg_priv *)ltcg;
+        struct gf100_ltcg_priv *priv = (struct gf100_ltcg_priv *)ltcg;
         int ret;
 
         ret = nouveau_mm_head(&priv->tags, 1, n, n, 1, pnode);
@@ -82,18 +73,18 @@ nvc0_ltcg_tags_alloc(struct nouveau_ltcg *ltcg, u32 n,
         return ret;
 }
 
-static void
-nvc0_ltcg_tags_free(struct nouveau_ltcg *ltcg, struct nouveau_mm_node **pnode)
+void
+gf100_ltcg_tags_free(struct nouveau_ltcg *ltcg, struct nouveau_mm_node **pnode)
 {
-        struct nvc0_ltcg_priv *priv = (struct nvc0_ltcg_priv *)ltcg;
+        struct gf100_ltcg_priv *priv = (struct gf100_ltcg_priv *)ltcg;
 
         nouveau_mm_free(&priv->tags, pnode);
 }
 
 static void
-nvc0_ltcg_tags_clear(struct nouveau_ltcg *ltcg, u32 first, u32 count)
+gf100_ltcg_tags_clear(struct nouveau_ltcg *ltcg, u32 first, u32 count)
 {
-        struct nvc0_ltcg_priv *priv = (struct nvc0_ltcg_priv *)ltcg;
+        struct gf100_ltcg_priv *priv = (struct gf100_ltcg_priv *)ltcg;
         u32 last = first + count - 1;
         int p, i;
 
@@ -104,16 +95,16 @@ nvc0_ltcg_tags_clear(struct nouveau_ltcg *ltcg, u32 first, u32 count)
         nv_wr32(priv, 0x17e8c8, 0x4); /* trigger clear */
 
         /* wait until it's finished with clearing */
-        for (p = 0; p < priv->part_nr; ++p) {
-                for (i = 0; i < priv->subp_nr; ++i)
+        for (p = 0; p < priv->ltc_nr; ++p) {
+                for (i = 0; i < priv->lts_nr; ++i)
                         nv_wait(priv, 0x1410c8 + p * 0x2000 + i * 0x400, ~0, 0);
         }
 }
 
 /* TODO: Figure out tag memory details and drop the over-cautious allocation.
  */
-static int
-nvc0_ltcg_init_tag_ram(struct nouveau_fb *pfb, struct nvc0_ltcg_priv *priv)
+int
+gf100_ltcg_init_tag_ram(struct nouveau_fb *pfb, struct gf100_ltcg_priv *priv)
 {
         u32 tag_size, tag_margin, tag_align;
         int ret;
@@ -124,7 +115,7 @@ nvc0_ltcg_init_tag_ram(struct nouveau_fb *pfb, struct nvc0_ltcg_priv *priv)
                 priv->num_tags = 1 << 17; /* we have 17 bits in PTE */
         priv->num_tags = (priv->num_tags + 63) & ~63; /* round up to 64 */
 
-        tag_align = priv->part_nr * 0x800;
+        tag_align = priv->ltc_nr * 0x800;
         tag_margin = (tag_align < 0x6000) ? 0x6000 : tag_align;
 
         /* 4 part 4 sub: 0x2000 bytes for 56 tags */
@@ -157,11 +148,11 @@ nvc0_ltcg_init_tag_ram(struct nouveau_fb *pfb, struct nvc0_ltcg_priv *priv)
 }
 
 static int
-nvc0_ltcg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+gf100_ltcg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
                struct nouveau_oclass *oclass, void *data, u32 size,
                struct nouveau_object **pobject)
 {
-        struct nvc0_ltcg_priv *priv;
+        struct gf100_ltcg_priv *priv;
         struct nouveau_fb *pfb = nouveau_fb(parent);
         u32 parts, mask;
         int ret, i;
@@ -175,27 +166,27 @@ nvc0_ltcg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
         mask = nv_rd32(priv, 0x022554);
         for (i = 0; i < parts; i++) {
                 if (!(mask & (1 << i)))
-                        priv->part_nr++;
+                        priv->ltc_nr++;
         }
-        priv->subp_nr = nv_rd32(priv, 0x17e8dc) >> 28;
+        priv->lts_nr = nv_rd32(priv, 0x17e8dc) >> 28;
 
-        ret = nvc0_ltcg_init_tag_ram(pfb, priv);
+        ret = gf100_ltcg_init_tag_ram(pfb, priv);
         if (ret)
                 return ret;
 
-        priv->base.tags_alloc = nvc0_ltcg_tags_alloc;
-        priv->base.tags_free  = nvc0_ltcg_tags_free;
-        priv->base.tags_clear = nvc0_ltcg_tags_clear;
+        priv->base.tags_alloc = gf100_ltcg_tags_alloc;
+        priv->base.tags_free  = gf100_ltcg_tags_free;
+        priv->base.tags_clear = gf100_ltcg_tags_clear;
 
-        nv_subdev(priv)->intr = nvc0_ltcg_intr;
+        nv_subdev(priv)->intr = gf100_ltcg_intr;
         return 0;
 }
 
-static void
-nvc0_ltcg_dtor(struct nouveau_object *object)
+void
+gf100_ltcg_dtor(struct nouveau_object *object)
 {
         struct nouveau_ltcg *ltcg = (struct nouveau_ltcg *)object;
-        struct nvc0_ltcg_priv *priv = (struct nvc0_ltcg_priv *)ltcg;
+        struct gf100_ltcg_priv *priv = (struct gf100_ltcg_priv *)ltcg;
         struct nouveau_fb *pfb = nouveau_fb(ltcg->base.base.parent);
 
         nouveau_mm_fini(&priv->tags);
@@ -205,10 +196,10 @@ nvc0_ltcg_dtor(struct nouveau_object *object)
 }
 
 static int
-nvc0_ltcg_init(struct nouveau_object *object)
+gf100_ltcg_init(struct nouveau_object *object)
 {
         struct nouveau_ltcg *ltcg = (struct nouveau_ltcg *)object;
-        struct nvc0_ltcg_priv *priv = (struct nvc0_ltcg_priv *)ltcg;
+        struct gf100_ltcg_priv *priv = (struct gf100_ltcg_priv *)ltcg;
         int ret;
 
         ret = nouveau_ltcg_init(ltcg);
@@ -216,20 +207,20 @@ nvc0_ltcg_init(struct nouveau_object *object)
                 return ret;
 
         nv_mask(priv, 0x17e820, 0x00100000, 0x00000000); /* INTR_EN &= ~0x10 */
-        nv_wr32(priv, 0x17e8d8, priv->part_nr);
+        nv_wr32(priv, 0x17e8d8, priv->ltc_nr);
         if (nv_device(ltcg)->card_type >= NV_E0)
-                nv_wr32(priv, 0x17e000, priv->part_nr);
+                nv_wr32(priv, 0x17e000, priv->ltc_nr);
         nv_wr32(priv, 0x17e8d4, priv->tag_base);
         return 0;
 }
 
-struct nouveau_oclass
-nvc0_ltcg_oclass = {
+struct nouveau_oclass *
+gf100_ltcg_oclass = &(struct nouveau_oclass) {
         .handle = NV_SUBDEV(LTCG, 0xc0),
         .ofuncs = &(struct nouveau_ofuncs) {
-                .ctor = nvc0_ltcg_ctor,
-                .dtor = nvc0_ltcg_dtor,
-                .init = nvc0_ltcg_init,
+                .ctor = gf100_ltcg_ctor,
+                .dtor = gf100_ltcg_dtor,
+                .init = gf100_ltcg_init,
                 .fini = _nouveau_ltcg_fini,
         },
 };
diff --git a/drivers/gpu/drm/nouveau/core/subdev/ltcg/gf100.h b/drivers/gpu/drm/nouveau/core/subdev/ltcg/gf100.h
new file mode 100644
index 000000000000..87b10b8412ea
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/ltcg/gf100.h
@@ -0,0 +1,21 @@
+#ifndef __NVKM_LTCG_PRIV_GF100_H__
+#define __NVKM_LTCG_PRIV_GF100_H__
+
+#include <subdev/ltcg.h>
+
+struct gf100_ltcg_priv {
+        struct nouveau_ltcg base;
+        u32 ltc_nr;
+        u32 lts_nr;
+        u32 num_tags;
+        u32 tag_base;
+        struct nouveau_mm tags;
+        struct nouveau_mm_node *tag_ram;
+};
+
+void gf100_ltcg_dtor(struct nouveau_object *);
+int  gf100_ltcg_init_tag_ram(struct nouveau_fb *, struct gf100_ltcg_priv *);
+int  gf100_ltcg_tags_alloc(struct nouveau_ltcg *, u32, struct nouveau_mm_node **);
+void gf100_ltcg_tags_free(struct nouveau_ltcg *, struct nouveau_mm_node **);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/ltcg/gm107.c b/drivers/gpu/drm/nouveau/core/subdev/ltcg/gm107.c
new file mode 100644
index 000000000000..e79d0e81de40
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/ltcg/gm107.c
@@ -0,0 +1,142 @@
+/*
+ * Copyright 2014 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include <subdev/fb.h>
+#include <subdev/timer.h>
+
+#include "gf100.h"
+
+static void
+gm107_ltcg_lts_isr(struct gf100_ltcg_priv *priv, int ltc, int lts)
+{
+        u32 base = 0x140000 + (ltc * 0x2000) + (lts * 0x400);
+        u32 stat = nv_rd32(priv, base + 0x00c);
+
+        if (stat) {
+                nv_info(priv, "LTC%d_LTS%d: 0x%08x\n", ltc, lts, stat);
+                nv_wr32(priv, base + 0x00c, stat);
+        }
+}
+
+static void
+gm107_ltcg_intr(struct nouveau_subdev *subdev)
+{
+        struct gf100_ltcg_priv *priv = (void *)subdev;
+        u32 mask;
+
+        mask = nv_rd32(priv, 0x00017c);
+        while (mask) {
+                u32 lts, ltc = __ffs(mask);
+                for (lts = 0; lts < priv->lts_nr; lts++)
+                        gm107_ltcg_lts_isr(priv, ltc, lts);
+                mask &= ~(1 << ltc);
+        }
+
+        /* we do something horribly wrong and upset PMFB a lot, so mask off
+         * interrupts from it after the first one until it's fixed
+         */
+        nv_mask(priv, 0x000640, 0x02000000, 0x00000000);
+}
+
+static void
+gm107_ltcg_tags_clear(struct nouveau_ltcg *ltcg, u32 first, u32 count)
+{
+        struct gf100_ltcg_priv *priv = (struct gf100_ltcg_priv *)ltcg;
+        u32 last = first + count - 1;
+        int p, i;
+
+        BUG_ON((first > last) || (last >= priv->num_tags));
+
+        nv_wr32(priv, 0x17e270, first);
+        nv_wr32(priv, 0x17e274, last);
+        nv_wr32(priv, 0x17e26c, 0x4); /* trigger clear */
+
+        /* wait until it's finished with clearing */
+        for (p = 0; p < priv->ltc_nr; ++p) {
+                for (i = 0; i < priv->lts_nr; ++i)
+                        nv_wait(priv, 0x14046c + p * 0x2000 + i * 0x200, ~0, 0);
+        }
+}
+
+static int
+gm107_ltcg_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+               struct nouveau_oclass *oclass, void *data, u32 size,
+               struct nouveau_object **pobject)
+{
+        struct gf100_ltcg_priv *priv;
+        struct nouveau_fb *pfb = nouveau_fb(parent);
+        u32 parts, mask;
+        int ret, i;
+
+        ret = nouveau_ltcg_create(parent, engine, oclass, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        parts = nv_rd32(priv, 0x022438);
+        mask = nv_rd32(priv, 0x021c14);
+        for (i = 0; i < parts; i++) {
+                if (!(mask & (1 << i)))
+                        priv->ltc_nr++;
+        }
+        priv->lts_nr = nv_rd32(priv, 0x17e280) >> 28;
+
+        ret = gf100_ltcg_init_tag_ram(pfb, priv);
+        if (ret)
+                return ret;
+
+        priv->base.tags_alloc = gf100_ltcg_tags_alloc;
+        priv->base.tags_free  = gf100_ltcg_tags_free;
+        priv->base.tags_clear = gm107_ltcg_tags_clear;
+
+        nv_subdev(priv)->intr = gm107_ltcg_intr;
+        return 0;
+}
+
+static int
+gm107_ltcg_init(struct nouveau_object *object)
+{
+        struct nouveau_ltcg *ltcg = (struct nouveau_ltcg *)object;
+        struct gf100_ltcg_priv *priv = (struct gf100_ltcg_priv *)ltcg;
+        int ret;
+
+        ret = nouveau_ltcg_init(ltcg);
+        if (ret)
+                return ret;
+
+        nv_wr32(priv, 0x17e27c, priv->ltc_nr);
+        nv_wr32(priv, 0x17e278, priv->tag_base);
+        return 0;
+}
+
+struct nouveau_oclass *
+gm107_ltcg_oclass = &(struct nouveau_oclass) {
+        .handle = NV_SUBDEV(LTCG, 0xff),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = gm107_ltcg_ctor,
+                .dtor = gf100_ltcg_dtor,
+                .init = gm107_ltcg_init,
+                .fini = _nouveau_ltcg_fini,
+        },
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mc/base.c b/drivers/gpu/drm/nouveau/core/subdev/mc/base.c
index b4b9943773bc..8a5555192fa5 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/mc/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/mc/base.c
@@ -93,7 +93,7 @@ _nouveau_mc_dtor(struct nouveau_object *object)
 {
         struct nouveau_device *device = nv_device(object);
         struct nouveau_mc *pmc = (void *)object;
-        free_irq(device->pdev->irq, pmc);
+        free_irq(pmc->irq, pmc);
         if (pmc->use_msi)
                 pci_disable_msi(device->pdev);
         nouveau_subdev_destroy(&pmc->base);
@@ -114,33 +114,44 @@ nouveau_mc_create_(struct nouveau_object *parent, struct nouveau_object *engine,
         if (ret)
                 return ret;
 
-        switch (device->pdev->device & 0x0ff0) {
-        case 0x00f0:
-        case 0x02e0:
-                /* BR02? NFI how these would be handled yet exactly */
-                break;
-        default:
-                switch (device->chipset) {
-                case 0xaa: break; /* reported broken, nv also disable it */
-                default:
-                        pmc->use_msi = true;
+        if (nv_device_is_pci(device))
+                switch (device->pdev->device & 0x0ff0) {
+                case 0x00f0:
+                case 0x02e0:
+                        /* BR02? NFI how these would be handled yet exactly */
                         break;
+                default:
+                        switch (device->chipset) {
+                        case 0xaa:
+                                /* reported broken, nv also disable it */
+                                break;
+                        default:
+                                pmc->use_msi = true;
+                                break;
                 }
-        }
 
-        pmc->use_msi = nouveau_boolopt(device->cfgopt, "NvMSI", pmc->use_msi);
-        if (pmc->use_msi && oclass->msi_rearm) {
-                pmc->use_msi = pci_enable_msi(device->pdev) == 0;
-                if (pmc->use_msi) {
-                        nv_info(pmc, "MSI interrupts enabled\n");
-                        oclass->msi_rearm(pmc);
+                pmc->use_msi = nouveau_boolopt(device->cfgopt, "NvMSI",
+                                               pmc->use_msi);
+
+                if (pmc->use_msi && oclass->msi_rearm) {
+                        pmc->use_msi = pci_enable_msi(device->pdev) == 0;
+                        if (pmc->use_msi) {
+                                nv_info(pmc, "MSI interrupts enabled\n");
+                                oclass->msi_rearm(pmc);
+                        }
+                } else {
+                        pmc->use_msi = false;
                 }
-        } else {
-                pmc->use_msi = false;
         }
 
-        ret = request_irq(device->pdev->irq, nouveau_mc_intr,
-                          IRQF_SHARED, "nouveau", pmc);
+        ret = nv_device_get_irq(device, true);
+        if (ret < 0)
+                return ret;
+        pmc->irq = ret;
+
+        ret = request_irq(pmc->irq, nouveau_mc_intr, IRQF_SHARED, "nouveau",
+                          pmc);
+
         if (ret < 0)
                 return ret;
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/mxm/base.c b/drivers/gpu/drm/nouveau/core/subdev/mxm/base.c
index 13c5af88a601..51fcf7960417 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/mxm/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/mxm/base.c
@@ -96,7 +96,7 @@ mxm_shadow_dsm(struct nouveau_mxm *mxm, u8 version)
         acpi_handle handle;
         int rev;
 
-        handle = ACPI_HANDLE(&device->pdev->dev);
+        handle = ACPI_HANDLE(nv_device_base(device));
         if (!handle)
                 return false;
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/base.c b/drivers/gpu/drm/nouveau/core/subdev/therm/base.c
index 80e584a1bd1c..9ad01da6eacb 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/base.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/base.c
@@ -110,16 +110,18 @@ nouveau_therm_update(struct nouveau_therm *therm, int mode)
                 poll = false;
                 break;
         case NOUVEAU_THERM_CTRL_AUTO:
-                if (priv->fan->bios.nr_fan_trip) {
+                switch(priv->fan->bios.fan_mode) {
+                case NVBIOS_THERM_FAN_TRIP:
                         duty = nouveau_therm_update_trip(therm);
-                } else
-                if (priv->fan->bios.linear_min_temp ||
-                    priv->fan->bios.linear_max_temp) {
+                        break;
+                case NVBIOS_THERM_FAN_LINEAR:
                         duty = nouveau_therm_update_linear(therm);
-                } else {
+                        break;
+                case NVBIOS_THERM_FAN_OTHER:
                         if (priv->cstate)
                                 duty = priv->cstate;
                         poll = false;
+                        break;
                 }
                 immd = false;
                 break;
@@ -179,7 +181,7 @@ nouveau_therm_fan_mode(struct nouveau_therm *therm, int mode)
 
         /* do not allow automatic fan management if the thermal sensor is
          * not available */
-        if (priv->mode == NOUVEAU_THERM_CTRL_AUTO && therm->temp_get(therm) < 0)
+        if (mode == NOUVEAU_THERM_CTRL_AUTO && therm->temp_get(therm) < 0)
                 return -EINVAL;
 
         if (priv->mode == mode)
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/fan.c b/drivers/gpu/drm/nouveau/core/subdev/therm/fan.c
index 95f6129eeede..016990a8252c 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/fan.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/fan.c
@@ -54,8 +54,10 @@ nouveau_fan_update(struct nouveau_fan *fan, bool immediate, int target)
 
         /* check that we're not already at the target duty cycle */
         duty = fan->get(therm);
-        if (duty == target)
-                goto done;
+        if (duty == target) {
+                spin_unlock_irqrestore(&fan->lock, flags);
+                return 0;
+        }
 
         /* smooth out the fanspeed increase/decrease */
         if (!immediate && duty >= 0) {
@@ -73,8 +75,15 @@ nouveau_fan_update(struct nouveau_fan *fan, bool immediate, int target)
 
         nv_debug(therm, "FAN update: %d\n", duty);
         ret = fan->set(therm, duty);
-        if (ret)
-                goto done;
+        if (ret) {
+                spin_unlock_irqrestore(&fan->lock, flags);
+                return ret;
+        }
+
+        /* fan speed updated, drop the fan lock before grabbing the
+         * alarm-scheduling lock and risking a deadlock
+         */
+        spin_unlock_irqrestore(&fan->lock, flags);
 
         /* schedule next fan update, if not at target speed already */
         if (list_empty(&fan->alarm.head) && target != duty) {
@@ -92,8 +101,6 @@ nouveau_fan_update(struct nouveau_fan *fan, bool immediate, int target)
                 ptimer->alarm(ptimer, delay * 1000 * 1000, &fan->alarm);
         }
 
-done:
-        spin_unlock_irqrestore(&fan->lock, flags);
         return ret;
 }
 
@@ -185,11 +192,8 @@ nouveau_therm_fan_set_defaults(struct nouveau_therm *therm)
         priv->fan->bios.max_duty = 100;
         priv->fan->bios.bump_period = 500;
         priv->fan->bios.slow_down_period = 2000;
-/*XXX: talk to mupuf */
-#if 0
         priv->fan->bios.linear_min_temp = 40;
         priv->fan->bios.linear_max_temp = 85;
-#endif
 }
 
 static void
@@ -235,7 +239,8 @@ nouveau_therm_fan_ctor(struct nouveau_therm *therm)
         /* attempt to locate a drivable fan, and determine control method */
         ret = gpio->find(gpio, 0, DCB_GPIO_FAN, 0xff, &func);
         if (ret == 0) {
-                if (func.log[0] & DCB_GPIO_LOG_DIR_IN) {
+                /* FIXME: is this really the place to perform such checks ? */
+                if (func.line != 16 && func.log[0] & DCB_GPIO_LOG_DIR_IN) {
                         nv_debug(therm, "GPIO_FAN is in input mode\n");
                         ret = -EINVAL;
                 } else {
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/fanpwm.c b/drivers/gpu/drm/nouveau/core/subdev/therm/fanpwm.c
index 5f71db8e8992..9a5c07340263 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/fanpwm.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/fanpwm.c
@@ -67,7 +67,7 @@ nouveau_fanpwm_set(struct nouveau_therm *therm, int percent)
         if (priv->base.bios.pwm_freq) {
                 divs = 1;
                 if (therm->pwm_clock)
-                        divs = therm->pwm_clock(therm);
+                        divs = therm->pwm_clock(therm, priv->func.line);
                 divs /= priv->base.bios.pwm_freq;
         }
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/nv50.c b/drivers/gpu/drm/nouveau/core/subdev/therm/nv50.c
index 8cf7597a2182..321db927d638 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/nv50.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/nv50.c
@@ -93,7 +93,7 @@ nv50_fan_pwm_set(struct nouveau_therm *therm, int line, u32 divs, u32 duty)
 }
 
 int
-nv50_fan_pwm_clock(struct nouveau_therm *therm)
+nv50_fan_pwm_clock(struct nouveau_therm *therm, int line)
 {
         int chipset = nv_device(therm)->chipset;
         int crystal = nv_device(therm)->crystal;
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/nvd0.c b/drivers/gpu/drm/nouveau/core/subdev/therm/nvd0.c
index 4dd4f81ae873..43fec17ea540 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/nvd0.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/nvd0.c
@@ -32,10 +32,12 @@ static int
 pwm_info(struct nouveau_therm *therm, int line)
 {
         u32 gpio = nv_rd32(therm, 0x00d610 + (line * 0x04));
+
         switch (gpio & 0x000000c0) {
         case 0x00000000: /* normal mode, possibly pwm forced off by us */
         case 0x00000040: /* nvio special */
                 switch (gpio & 0x0000001f) {
+                case 0x00: return 2;
                 case 0x19: return 1;
                 case 0x1c: return 0;
                 default:
@@ -56,8 +58,9 @@ nvd0_fan_pwm_ctrl(struct nouveau_therm *therm, int line, bool enable)
         int indx = pwm_info(therm, line);
         if (indx < 0)
                 return indx;
-
-        nv_mask(therm, 0x00d610 + (line * 0x04), 0x000000c0, data);
+        else if (indx < 2)
+                nv_mask(therm, 0x00d610 + (line * 0x04), 0x000000c0, data);
+        /* nothing to do for indx == 2, it seems hardwired to PTHERM */
         return 0;
 }
 
@@ -67,10 +70,15 @@ nvd0_fan_pwm_get(struct nouveau_therm *therm, int line, u32 *divs, u32 *duty)
         int indx = pwm_info(therm, line);
         if (indx < 0)
                 return indx;
-
-        if (nv_rd32(therm, 0x00d610 + (line * 0x04)) & 0x00000040) {
-                *divs = nv_rd32(therm, 0x00e114 + (indx * 8));
-                *duty = nv_rd32(therm, 0x00e118 + (indx * 8));
+        else if (indx < 2) {
+                if (nv_rd32(therm, 0x00d610 + (line * 0x04)) & 0x00000040) {
+                        *divs = nv_rd32(therm, 0x00e114 + (indx * 8));
+                        *duty = nv_rd32(therm, 0x00e118 + (indx * 8));
+                        return 0;
+                }
+        } else if (indx == 2) {
+                *divs = nv_rd32(therm, 0x0200d8) & 0x1fff;
+                *duty = nv_rd32(therm, 0x0200dc) & 0x1fff;
                 return 0;
         }
 
@@ -83,16 +91,26 @@ nvd0_fan_pwm_set(struct nouveau_therm *therm, int line, u32 divs, u32 duty)
         int indx = pwm_info(therm, line);
         if (indx < 0)
                 return indx;
-
-        nv_wr32(therm, 0x00e114 + (indx * 8), divs);
-        nv_wr32(therm, 0x00e118 + (indx * 8), duty | 0x80000000);
+        else if (indx < 2) {
+                nv_wr32(therm, 0x00e114 + (indx * 8), divs);
+                nv_wr32(therm, 0x00e118 + (indx * 8), duty | 0x80000000);
+        } else if (indx == 2) {
+                nv_mask(therm, 0x0200d8, 0x1fff, divs); /* keep the high bits */
+                nv_wr32(therm, 0x0200dc, duty | 0x40000000);
+        }
         return 0;
 }
 
 static int
-nvd0_fan_pwm_clock(struct nouveau_therm *therm)
+nvd0_fan_pwm_clock(struct nouveau_therm *therm, int line)
 {
-        return (nv_device(therm)->crystal * 1000) / 20;
+        int indx = pwm_info(therm, line);
+        if (indx < 0)
+                return 0;
+        else if (indx < 2)
+                return (nv_device(therm)->crystal * 1000) / 20;
+        else
+                return nv_device(therm)->crystal * 1000 / 10;
 }
 
 static int
diff --git a/drivers/gpu/drm/nouveau/core/subdev/therm/priv.h b/drivers/gpu/drm/nouveau/core/subdev/therm/priv.h
index 96f8f95693ce..916fca5c7816 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/therm/priv.h
+++ b/drivers/gpu/drm/nouveau/core/subdev/therm/priv.h
@@ -143,7 +143,7 @@ void nv40_therm_intr(struct nouveau_subdev *);
 int nv50_fan_pwm_ctrl(struct nouveau_therm *, int, bool);
 int nv50_fan_pwm_get(struct nouveau_therm *, int, u32 *, u32 *);
 int nv50_fan_pwm_set(struct nouveau_therm *, int, u32, u32);
-int nv50_fan_pwm_clock(struct nouveau_therm *);
+int nv50_fan_pwm_clock(struct nouveau_therm *, int);
 int nv84_temp_get(struct nouveau_therm *therm);
 int nv84_therm_fini(struct nouveau_object *object, bool suspend);
 
diff --git a/drivers/gpu/drm/nouveau/core/subdev/timer/gk20a.c b/drivers/gpu/drm/nouveau/core/subdev/timer/gk20a.c
new file mode 100644
index 000000000000..37484db1f7fc
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/timer/gk20a.c
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+
+#include "nv04.h"
+
+static int
+gk20a_timer_init(struct nouveau_object *object)
+{
+        struct nv04_timer_priv *priv = (void *)object;
+        u32 hi = upper_32_bits(priv->suspend_time);
+        u32 lo = lower_32_bits(priv->suspend_time);
+        int ret;
+
+        ret = nouveau_timer_init(&priv->base);
+        if (ret)
+                return ret;
+
+        nv_debug(priv, "time low        : 0x%08x\n", lo);
+        nv_debug(priv, "time high       : 0x%08x\n", hi);
+
+        /* restore the time before suspend */
+        nv_wr32(priv, NV04_PTIMER_TIME_1, hi);
+        nv_wr32(priv, NV04_PTIMER_TIME_0, lo);
+        return 0;
+}
+
+struct nouveau_oclass
+gk20a_timer_oclass = {
+        .handle = NV_SUBDEV(TIMER, 0xff),
+        .ofuncs = &(struct nouveau_ofuncs) {
+                .ctor = nv04_timer_ctor,
+                .dtor = nv04_timer_dtor,
+                .init = gk20a_timer_init,
+                .fini = nv04_timer_fini,
+        }
+};
diff --git a/drivers/gpu/drm/nouveau/core/subdev/timer/nv04.c b/drivers/gpu/drm/nouveau/core/subdev/timer/nv04.c
index c0bdd10358d7..240ed0b983a9 100644
--- a/drivers/gpu/drm/nouveau/core/subdev/timer/nv04.c
+++ b/drivers/gpu/drm/nouveau/core/subdev/timer/nv04.c
@@ -22,22 +22,7 @@
  * Authors: Ben Skeggs
  */
 
-#include <subdev/timer.h>
-
-#define NV04_PTIMER_INTR_0      0x009100
-#define NV04_PTIMER_INTR_EN_0   0x009140
-#define NV04_PTIMER_NUMERATOR   0x009200
-#define NV04_PTIMER_DENOMINATOR 0x009210
-#define NV04_PTIMER_TIME_0      0x009400
-#define NV04_PTIMER_TIME_1      0x009410
-#define NV04_PTIMER_ALARM_0     0x009420
-
-struct nv04_timer_priv {
-        struct nouveau_timer base;
-        struct list_head alarms;
-        spinlock_t lock;
-        u64 suspend_time;
-};
+#include "nv04.h"
 
 static u64
 nv04_timer_read(struct nouveau_timer *ptimer)
@@ -142,35 +127,14 @@ nv04_timer_intr(struct nouveau_subdev *subdev)
         }
 }
 
-static int
-nv04_timer_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
-                struct nouveau_oclass *oclass, void *data, u32 size,
-                struct nouveau_object **pobject)
-{
-        struct nv04_timer_priv *priv;
-        int ret;
-
-        ret = nouveau_timer_create(parent, engine, oclass, &priv);
-        *pobject = nv_object(priv);
-        if (ret)
-                return ret;
-
-        priv->base.base.intr = nv04_timer_intr;
-        priv->base.read = nv04_timer_read;
-        priv->base.alarm = nv04_timer_alarm;
-        priv->base.alarm_cancel = nv04_timer_alarm_cancel;
-        priv->suspend_time = 0;
-
-        INIT_LIST_HEAD(&priv->alarms);
-        spin_lock_init(&priv->lock);
-        return 0;
-}
-
-static void
-nv04_timer_dtor(struct nouveau_object *object)
+int
+nv04_timer_fini(struct nouveau_object *object, bool suspend)
 {
         struct nv04_timer_priv *priv = (void *)object;
-        return nouveau_timer_destroy(&priv->base);
+        if (suspend)
+                priv->suspend_time = nv04_timer_read(&priv->base);
+        nv_wr32(priv, NV04_PTIMER_INTR_EN_0, 0x00000000);
+        return nouveau_timer_fini(&priv->base, suspend);
 }
 
 static int
@@ -257,14 +221,35 @@ nv04_timer_init(struct nouveau_object *object)
         return 0;
 }
 
-static int
-nv04_timer_fini(struct nouveau_object *object, bool suspend)
+void
+nv04_timer_dtor(struct nouveau_object *object)
 {
         struct nv04_timer_priv *priv = (void *)object;
-        if (suspend)
-                priv->suspend_time = nv04_timer_read(&priv->base);
-        nv_wr32(priv, NV04_PTIMER_INTR_EN_0, 0x00000000);
-        return nouveau_timer_fini(&priv->base, suspend);
+        return nouveau_timer_destroy(&priv->base);
+}
+
+int
+nv04_timer_ctor(struct nouveau_object *parent, struct nouveau_object *engine,
+                struct nouveau_oclass *oclass, void *data, u32 size,
+                struct nouveau_object **pobject)
+{
+        struct nv04_timer_priv *priv;
+        int ret;
+
+        ret = nouveau_timer_create(parent, engine, oclass, &priv);
+        *pobject = nv_object(priv);
+        if (ret)
+                return ret;
+
+        priv->base.base.intr = nv04_timer_intr;
+        priv->base.read = nv04_timer_read;
+        priv->base.alarm = nv04_timer_alarm;
+        priv->base.alarm_cancel = nv04_timer_alarm_cancel;
+        priv->suspend_time = 0;
+
+        INIT_LIST_HEAD(&priv->alarms);
+        spin_lock_init(&priv->lock);
+        return 0;
 }
 
 struct nouveau_oclass
diff --git a/drivers/gpu/drm/nouveau/core/subdev/timer/nv04.h b/drivers/gpu/drm/nouveau/core/subdev/timer/nv04.h
new file mode 100644
index 000000000000..4bc152697c37
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/timer/nv04.h
@@ -0,0 +1,27 @@
+#ifndef __NVKM_TIMER_NV04_H__
+#define __NVKM_TIMER_NV04_H__
+
+#include "priv.h"
+
+#define NV04_PTIMER_INTR_0      0x009100
+#define NV04_PTIMER_INTR_EN_0   0x009140
+#define NV04_PTIMER_NUMERATOR   0x009200
+#define NV04_PTIMER_DENOMINATOR 0x009210
+#define NV04_PTIMER_TIME_0      0x009400
+#define NV04_PTIMER_TIME_1      0x009410
+#define NV04_PTIMER_ALARM_0     0x009420
+
+struct nv04_timer_priv {
+        struct nouveau_timer base;
+        struct list_head alarms;
+        spinlock_t lock;
+        u64 suspend_time;
+};
+
+int  nv04_timer_ctor(struct nouveau_object *, struct nouveau_object *,
+                     struct nouveau_oclass *, void *, u32,
+                     struct nouveau_object **);
+void nv04_timer_dtor(struct nouveau_object *);
+int  nv04_timer_fini(struct nouveau_object *, bool);
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/core/subdev/timer/priv.h b/drivers/gpu/drm/nouveau/core/subdev/timer/priv.h
new file mode 100644
index 000000000000..799dae3f2300
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/core/subdev/timer/priv.h
@@ -0,0 +1,6 @@
+#ifndef __NVKM_TIMER_PRIV_H__
+#define __NVKM_TIMER_PRIV_H__
+
+#include <subdev/timer.h>
+
+#endif
diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
index 0e3270c3ffd2..41be3424c906 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
@@ -239,7 +239,7 @@ nv_crtc_mode_set_vga(struct drm_crtc *crtc, struct drm_display_mode *mode)
         struct drm_device *dev = crtc->dev;
         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
         struct nv04_crtc_reg *regp = &nv04_display(dev)->mode_reg.crtc_reg[nv_crtc->index];
-        struct drm_framebuffer *fb = crtc->fb;
+        struct drm_framebuffer *fb = crtc->primary->fb;
 
         /* Calculate our timings */
         int horizDisplay        = (mode->crtc_hdisplay >> 3)            - 1;
@@ -574,7 +574,7 @@ nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
                 regp->CRTC[NV_CIO_CRE_86] = 0x1;
         }
 
-        regp->CRTC[NV_CIO_CRE_PIXEL_INDEX] = (crtc->fb->depth + 1) / 8;
+        regp->CRTC[NV_CIO_CRE_PIXEL_INDEX] = (crtc->primary->fb->depth + 1) / 8;
         /* Enable slaved mode (called MODE_TV in nv4ref.h) */
         if (lvds_output || tmds_output || tv_output)
                 regp->CRTC[NV_CIO_CRE_PIXEL_INDEX] |= (1 << 7);
@@ -588,7 +588,7 @@ nv_crtc_mode_set_regs(struct drm_crtc *crtc, struct drm_display_mode * mode)
         regp->ramdac_gen_ctrl = NV_PRAMDAC_GENERAL_CONTROL_BPC_8BITS |
                                 NV_PRAMDAC_GENERAL_CONTROL_VGA_STATE_SEL |
                                 NV_PRAMDAC_GENERAL_CONTROL_PIXMIX_ON;
-        if (crtc->fb->depth == 16)
+        if (crtc->primary->fb->depth == 16)
                 regp->ramdac_gen_ctrl |= NV_PRAMDAC_GENERAL_CONTROL_ALT_MODE_SEL;
         if (nv_device(drm->device)->chipset >= 0x11)
                 regp->ramdac_gen_ctrl |= NV_PRAMDAC_GENERAL_CONTROL_PIPE_LONG;
@@ -609,7 +609,7 @@ static int
 nv_crtc_swap_fbs(struct drm_crtc *crtc, struct drm_framebuffer *old_fb)
 {
         struct nv04_display *disp = nv04_display(crtc->dev);
-        struct nouveau_framebuffer *nvfb = nouveau_framebuffer(crtc->fb);
+        struct nouveau_framebuffer *nvfb = nouveau_framebuffer(crtc->primary->fb);
         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
         int ret;
 
@@ -808,7 +808,7 @@ nv_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b, uint32_t start,
          * mark the lut values as dirty by setting depth==0, and it'll be
          * uploaded on the first mode_set_base()
          */
-        if (!nv_crtc->base.fb) {
+        if (!nv_crtc->base.primary->fb) {
                 nv_crtc->lut.depth = 0;
                 return;
         }
@@ -832,7 +832,7 @@ nv04_crtc_do_mode_set_base(struct drm_crtc *crtc,
         NV_DEBUG(drm, "index %d\n", nv_crtc->index);
 
         /* no fb bound */
-        if (!atomic && !crtc->fb) {
+        if (!atomic && !crtc->primary->fb) {
                 NV_DEBUG(drm, "No FB bound\n");
                 return 0;
         }
@@ -844,8 +844,8 @@ nv04_crtc_do_mode_set_base(struct drm_crtc *crtc,
                 drm_fb = passed_fb;
                 fb = nouveau_framebuffer(passed_fb);
         } else {
-                drm_fb = crtc->fb;
-                fb = nouveau_framebuffer(crtc->fb);
+                drm_fb = crtc->primary->fb;
+                fb = nouveau_framebuffer(crtc->primary->fb);
         }
 
         nv_crtc->fb.offset = fb->nvbo->bo.offset;
@@ -857,9 +857,9 @@ nv04_crtc_do_mode_set_base(struct drm_crtc *crtc,
 
         /* Update the framebuffer format. */
         regp->CRTC[NV_CIO_CRE_PIXEL_INDEX] &= ~3;
-        regp->CRTC[NV_CIO_CRE_PIXEL_INDEX] |= (crtc->fb->depth + 1) / 8;
+        regp->CRTC[NV_CIO_CRE_PIXEL_INDEX] |= (crtc->primary->fb->depth + 1) / 8;
         regp->ramdac_gen_ctrl &= ~NV_PRAMDAC_GENERAL_CONTROL_ALT_MODE_SEL;
-        if (crtc->fb->depth == 16)
+        if (crtc->primary->fb->depth == 16)
                 regp->ramdac_gen_ctrl |= NV_PRAMDAC_GENERAL_CONTROL_ALT_MODE_SEL;
         crtc_wr_cio_state(crtc, regp, NV_CIO_CRE_PIXEL_INDEX);
         NVWriteRAMDAC(dev, nv_crtc->index, NV_PRAMDAC_GENERAL_CONTROL,
@@ -1048,7 +1048,7 @@ nouveau_crtc_set_config(struct drm_mode_set *set)
 
         /* get a pm reference here */
         ret = pm_runtime_get_sync(dev->dev);
-        if (ret < 0)
+        if (ret < 0 && ret != -EACCES)
                 return ret;
 
         ret = drm_crtc_helper_set_config(set);
diff --git a/drivers/gpu/drm/nouveau/dispnv04/dfp.c b/drivers/gpu/drm/nouveau/dispnv04/dfp.c
index 7fdc51e2a571..a2d669b4acf2 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/dfp.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/dfp.c
@@ -415,7 +415,7 @@ static void nv04_dfp_mode_set(struct drm_encoder *encoder,
         /* Output property. */
         if ((nv_connector->dithering_mode == DITHERING_MODE_ON) ||
             (nv_connector->dithering_mode == DITHERING_MODE_AUTO &&
-             encoder->crtc->fb->depth > connector->display_info.bpc * 3)) {
+             encoder->crtc->primary->fb->depth > connector->display_info.bpc * 3)) {
                 if (nv_device(drm->device)->chipset == 0x11)
                         regp->dither = savep->dither | 0x00010000;
                 else {
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c
index 900fae01793e..b13f441c6431 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.c
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c
@@ -97,6 +97,7 @@ nouveau_abi16_swclass(struct nouveau_drm *drm)
         case NV_C0:
         case NV_D0:
         case NV_E0:
+        case GM100:
                 return 0x906e;
         }
 
@@ -139,7 +140,7 @@ nouveau_abi16_chan_fini(struct nouveau_abi16 *abi16,
 
         /* destroy channel object, all children will be killed too */
         if (chan->chan) {
-                abi16->handles &= ~(1 << (chan->chan->handle & 0xffff));
+                abi16->handles &= ~(1ULL << (chan->chan->handle & 0xffff));
                 nouveau_channel_del(&chan->chan);
         }
 
@@ -179,12 +180,21 @@ nouveau_abi16_ioctl_getparam(ABI16_IOCTL_ARGS)
                 getparam->value = device->chipset;
                 break;
         case NOUVEAU_GETPARAM_PCI_VENDOR:
-                getparam->value = dev->pdev->vendor;
+                if (nv_device_is_pci(device))
+                        getparam->value = dev->pdev->vendor;
+                else
+                        getparam->value = 0;
                 break;
         case NOUVEAU_GETPARAM_PCI_DEVICE:
-                getparam->value = dev->pdev->device;
+                if (nv_device_is_pci(device))
+                        getparam->value = dev->pdev->device;
+                else
+                        getparam->value = 0;
                 break;
         case NOUVEAU_GETPARAM_BUS_TYPE:
+                if (!nv_device_is_pci(device))
+                        getparam->value = 3;
+                else
                 if (drm_pci_device_is_agp(dev))
                         getparam->value = 0;
                 else
@@ -270,8 +280,8 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS)
                 return nouveau_abi16_put(abi16, -EINVAL);
 
         /* allocate "abi16 channel" data and make up a handle for it */
-        init->channel = ffsll(~abi16->handles);
-        if (!init->channel--)
+        init->channel = __ffs64(~abi16->handles);
+        if (~abi16->handles == 0)
                 return nouveau_abi16_put(abi16, -ENOSPC);
 
         chan = kzalloc(sizeof(*chan), GFP_KERNEL);
@@ -280,7 +290,7 @@ nouveau_abi16_ioctl_channel_alloc(ABI16_IOCTL_ARGS)
 
         INIT_LIST_HEAD(&chan->notifiers);
         list_add(&chan->head, &abi16->channels);
-        abi16->handles |= (1 << init->channel);
+        abi16->handles |= (1ULL << init->channel);
 
         /* create channel object and initialise dma and fence management */
         ret = nouveau_channel_new(drm, cli, NVDRM_DEVICE, NVDRM_CHAN |
diff --git a/drivers/gpu/drm/nouveau/nouveau_agp.c b/drivers/gpu/drm/nouveau/nouveau_agp.c
index 2953c4e91e1a..51666daddb94 100644
--- a/drivers/gpu/drm/nouveau/nouveau_agp.c
+++ b/drivers/gpu/drm/nouveau/nouveau_agp.c
@@ -75,7 +75,7 @@ nouveau_agp_enabled(struct nouveau_drm *drm)
 {
         struct drm_device *dev = drm->dev;
 
-        if (!drm_pci_device_is_agp(dev) || !dev->agp)
+        if (!dev->pdev || !drm_pci_device_is_agp(dev) || !dev->agp)
                 return false;
 
         if (drm->agp.stat == UNKNOWN) {
diff --git a/drivers/gpu/drm/nouveau/nouveau_bios.c b/drivers/gpu/drm/nouveau/nouveau_bios.c
index 4c3feaaa1037..8268a4ccac15 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bios.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bios.c
@@ -1474,9 +1474,12 @@ parse_dcb20_entry(struct drm_device *dev, struct dcb_table *dcb,
                 case 0:
                         entry->dpconf.link_bw = 162000;
                         break;
-                default:
+                case 1:
                         entry->dpconf.link_bw = 270000;
                         break;
+                default:
+                        entry->dpconf.link_bw = 540000;
+                        break;
                 }
                 switch ((conf & 0x0f000000) >> 24) {
                 case 0xf:
@@ -2069,6 +2072,10 @@ nouveau_bios_init(struct drm_device *dev)
         struct nvbios *bios = &drm->vbios;
         int ret;
 
+        /* only relevant for PCI devices */
+        if (!dev->pdev)
+                return 0;
+
         if (!NVInitVBIOS(dev))
                 return -ENODEV;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 4aed1714b9ab..b6dc85c614be 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -1255,7 +1255,7 @@ nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
                 /* fallthrough, tiled memory */
         case TTM_PL_VRAM:
                 mem->bus.offset = mem->start << PAGE_SHIFT;
-                mem->bus.base = pci_resource_start(dev->pdev, 1);
+                mem->bus.base = nv_device_resource_start(nouveau_dev(dev), 1);
                 mem->bus.is_iomem = true;
                 if (nv_device(drm->device)->card_type >= NV_50) {
                         struct nouveau_bar *bar = nouveau_bar(drm->device);
@@ -1293,7 +1293,7 @@ nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
         struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
         struct nouveau_bo *nvbo = nouveau_bo(bo);
         struct nouveau_device *device = nv_device(drm->device);
-        u32 mappable = pci_resource_len(device->pdev, 1) >> PAGE_SHIFT;
+        u32 mappable = nv_device_resource_len(device, 1) >> PAGE_SHIFT;
         int ret;
 
         /* as long as the bo isn't in vram, and isn't tiled, we've got
@@ -1331,6 +1331,7 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm)
 {
         struct ttm_dma_tt *ttm_dma = (void *)ttm;
         struct nouveau_drm *drm;
+        struct nouveau_device *device;
         struct drm_device *dev;
         unsigned i;
         int r;
@@ -1348,6 +1349,7 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm)
         }
 
         drm = nouveau_bdev(ttm->bdev);
+        device = nv_device(drm->device);
         dev = drm->dev;
 
 #if __OS_HAS_AGP
@@ -1368,13 +1370,12 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm)
         }
 
         for (i = 0; i < ttm->num_pages; i++) {
-                ttm_dma->dma_address[i] = pci_map_page(dev->pdev, ttm->pages[i],
-                                                   0, PAGE_SIZE,
-                                                   PCI_DMA_BIDIRECTIONAL);
-                if (pci_dma_mapping_error(dev->pdev, ttm_dma->dma_address[i])) {
+                ttm_dma->dma_address[i] = nv_device_map_page(device,
+                                                             ttm->pages[i]);
+                if (!ttm_dma->dma_address[i]) {
                         while (--i) {
-                                pci_unmap_page(dev->pdev, ttm_dma->dma_address[i],
-                                               PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+                                nv_device_unmap_page(device,
+                                                     ttm_dma->dma_address[i]);
                                 ttm_dma->dma_address[i] = 0;
                         }
                         ttm_pool_unpopulate(ttm);
@@ -1389,6 +1390,7 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
 {
         struct ttm_dma_tt *ttm_dma = (void *)ttm;
         struct nouveau_drm *drm;
+        struct nouveau_device *device;
         struct drm_device *dev;
         unsigned i;
         bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
@@ -1397,6 +1399,7 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
                 return;
 
         drm = nouveau_bdev(ttm->bdev);
+        device = nv_device(drm->device);
         dev = drm->dev;
 
 #if __OS_HAS_AGP
@@ -1415,8 +1418,7 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
 
         for (i = 0; i < ttm->num_pages; i++) {
                 if (ttm_dma->dma_address[i]) {
-                        pci_unmap_page(dev->pdev, ttm_dma->dma_address[i],
-                                       PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+                        nv_device_unmap_page(device, ttm_dma->dma_address[i]);
                 }
         }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c
index cc5152be2cf1..ccb6b452d6d0 100644
--- a/drivers/gpu/drm/nouveau/nouveau_chan.c
+++ b/drivers/gpu/drm/nouveau/nouveau_chan.c
@@ -154,7 +154,7 @@ nouveau_channel_prep(struct nouveau_drm *drm, struct nouveau_cli *cli,
                          * nfi why this exists, it came from the -nv ddx.
                          */
                         args.flags = NV_DMA_TARGET_PCI | NV_DMA_ACCESS_RDWR;
-                        args.start = pci_resource_start(device->pdev, 1);
+                        args.start = nv_device_resource_start(device, 1);
                         args.limit = args.start + limit;
                 } else {
                         args.flags = NV_DMA_TARGET_VRAM | NV_DMA_ACCESS_RDWR;
diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c
index 1674882d60d5..d07ce028af51 100644
--- a/drivers/gpu/drm/nouveau/nouveau_connector.c
+++ b/drivers/gpu/drm/nouveau/nouveau_connector.c
@@ -255,7 +255,7 @@ nouveau_connector_detect(struct drm_connector *connector, bool force)
         }
 
         ret = pm_runtime_get_sync(connector->dev->dev);
-        if (ret < 0)
+        if (ret < 0 && ret != -EACCES)
                 return conn_status;
 
         i2c = nouveau_connector_ddc_detect(connector, &nv_encoder);
@@ -960,7 +960,8 @@ drm_conntype_from_dcb(enum dcb_connector_type dcb)
         case DCB_CONNECTOR_DP       : return DRM_MODE_CONNECTOR_DisplayPort;
         case DCB_CONNECTOR_eDP      : return DRM_MODE_CONNECTOR_eDP;
         case DCB_CONNECTOR_HDMI_0   :
-        case DCB_CONNECTOR_HDMI_1   : return DRM_MODE_CONNECTOR_HDMIA;
+        case DCB_CONNECTOR_HDMI_1   :
+        case DCB_CONNECTOR_HDMI_C   : return DRM_MODE_CONNECTOR_HDMIA;
         default:
                 break;
         }
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index 24011596af43..3ff030dc1ee3 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -105,7 +105,7 @@ nouveau_display_scanoutpos_head(struct drm_crtc *crtc, int *vpos, int *hpos,
                 if (retry) ndelay(crtc->linedur_ns);
         } while (retry--);
 
-        *hpos = calc(args.hblanks, args.hblanke, args.htotal, args.hline);
+        *hpos = args.hline;
         *vpos = calc(args.vblanks, args.vblanke, args.vtotal, args.vline);
         if (stime) *stime = ns_to_ktime(args.time[0]);
         if (etime) *etime = ns_to_ktime(args.time[1]);
@@ -419,6 +419,7 @@ int
 nouveau_display_create(struct drm_device *dev)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
+        struct nouveau_device *device = nouveau_dev(dev);
         struct nouveau_display *disp;
         int ret, gen;
 
@@ -459,7 +460,7 @@ nouveau_display_create(struct drm_device *dev)
         }
 
         dev->mode_config.funcs = &nouveau_mode_config_funcs;
-        dev->mode_config.fb_base = pci_resource_start(dev->pdev, 1);
+        dev->mode_config.fb_base = nv_device_resource_start(device, 1);
 
         dev->mode_config.min_width = 0;
         dev->mode_config.min_height = 0;
@@ -488,6 +489,7 @@ nouveau_display_create(struct drm_device *dev)
 
         if (drm->vbios.dcb.entries) {
                 static const u16 oclass[] = {
+                        GM107_DISP_CLASS,
                         NVF0_DISP_CLASS,
                         NVE0_DISP_CLASS,
                         NVD0_DISP_CLASS,
@@ -569,7 +571,7 @@ nouveau_display_suspend(struct drm_device *dev)
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                 struct nouveau_framebuffer *nouveau_fb;
 
-                nouveau_fb = nouveau_framebuffer(crtc->fb);
+                nouveau_fb = nouveau_framebuffer(crtc->primary->fb);
                 if (!nouveau_fb || !nouveau_fb->nvbo)
                         continue;
 
@@ -596,7 +598,7 @@ nouveau_display_repin(struct drm_device *dev)
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
                 struct nouveau_framebuffer *nouveau_fb;
 
-                nouveau_fb = nouveau_framebuffer(crtc->fb);
+                nouveau_fb = nouveau_framebuffer(crtc->primary->fb);
                 if (!nouveau_fb || !nouveau_fb->nvbo)
                         continue;
 
@@ -693,7 +695,7 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         const int swap_interval = (flags & DRM_MODE_PAGE_FLIP_ASYNC) ? 0 : 1;
         struct drm_device *dev = crtc->dev;
         struct nouveau_drm *drm = nouveau_drm(dev);
-        struct nouveau_bo *old_bo = nouveau_framebuffer(crtc->fb)->nvbo;
+        struct nouveau_bo *old_bo = nouveau_framebuffer(crtc->primary->fb)->nvbo;
         struct nouveau_bo *new_bo = nouveau_framebuffer(fb)->nvbo;
         struct nouveau_page_flip_state *s;
         struct nouveau_channel *chan = drm->channel;
@@ -767,7 +769,7 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
                 goto fail_unreserve;
 
         /* Update the crtc struct and cleanup */
-        crtc->fb = fb;
+        crtc->primary->fb = fb;
 
         nouveau_bo_fence(old_bo, fence);
         ttm_bo_unreserve(&old_bo->bo);
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 4ee702ac8907..ddd83756b9a2 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -33,6 +33,7 @@
 #include <core/client.h>
 #include <core/gpuobj.h>
 #include <core/class.h>
+#include <core/option.h>
 
 #include <engine/device.h>
 #include <engine/disp.h>
@@ -81,7 +82,7 @@ module_param_named(runpm, nouveau_runtime_pm, int, 0400);
 static struct drm_driver driver;
 
 static u64
-nouveau_name(struct pci_dev *pdev)
+nouveau_pci_name(struct pci_dev *pdev)
 {
         u64 name = (u64)pci_domain_nr(pdev->bus) << 32;
         name |= pdev->bus->number << 16;
@@ -89,15 +90,30 @@ nouveau_name(struct pci_dev *pdev)
         return name | PCI_FUNC(pdev->devfn);
 }
 
+static u64
+nouveau_platform_name(struct platform_device *platformdev)
+{
+        return platformdev->id;
+}
+
+static u64
+nouveau_name(struct drm_device *dev)
+{
+        if (dev->pdev)
+                return nouveau_pci_name(dev->pdev);
+        else
+                return nouveau_platform_name(dev->platformdev);
+}
+
 static int
-nouveau_cli_create(struct pci_dev *pdev, const char *name,
+nouveau_cli_create(u64 name, const char *sname,
                    int size, void **pcli)
 {
         struct nouveau_cli *cli;
         int ret;
 
         *pcli = NULL;
-        ret = nouveau_client_create_(name, nouveau_name(pdev), nouveau_config,
+        ret = nouveau_client_create_(sname, name, nouveau_config,
                                      nouveau_debug, size, pcli);
         cli = *pcli;
         if (ret) {
@@ -281,7 +297,8 @@ static int nouveau_drm_probe(struct pci_dev *pdev,
         remove_conflicting_framebuffers(aper, "nouveaufb", boot);
         kfree(aper);
 
-        ret = nouveau_device_create(pdev, nouveau_name(pdev), pci_name(pdev),
+        ret = nouveau_device_create(pdev, NOUVEAU_BUS_PCI,
+                                    nouveau_pci_name(pdev), pci_name(pdev),
                                     nouveau_config, nouveau_debug, &device);
         if (ret)
                 return ret;
@@ -300,22 +317,27 @@ static int nouveau_drm_probe(struct pci_dev *pdev,
 #define PCI_CLASS_MULTIMEDIA_HD_AUDIO 0x0403
 
 static void
-nouveau_get_hdmi_dev(struct drm_device *dev)
+nouveau_get_hdmi_dev(struct nouveau_drm *drm)
 {
-        struct nouveau_drm *drm = dev->dev_private;
-        struct pci_dev *pdev = dev->pdev;
+        struct pci_dev *pdev = drm->dev->pdev;
+
+        if (!pdev) {
+                DRM_INFO("not a PCI device; no HDMI\n");
+                drm->hdmi_device = NULL;
+                return;
+        }
 
         /* subfunction one is a hdmi audio device? */
         drm->hdmi_device = pci_get_bus_and_slot((unsigned int)pdev->bus->number,
                                                 PCI_DEVFN(PCI_SLOT(pdev->devfn), 1));
 
         if (!drm->hdmi_device) {
-                DRM_INFO("hdmi device  not found %d %d %d\n", pdev->bus->number, PCI_SLOT(pdev->devfn), 1);
+                NV_DEBUG(drm, "hdmi device not found %d %d %d\n", pdev->bus->number, PCI_SLOT(pdev->devfn), 1);
                 return;
         }
 
         if ((drm->hdmi_device->class >> 8) != PCI_CLASS_MULTIMEDIA_HD_AUDIO) {
-                DRM_INFO("possible hdmi device  not audio %d\n", drm->hdmi_device->class);
+                NV_DEBUG(drm, "possible hdmi device not audio %d\n", drm->hdmi_device->class);
                 pci_dev_put(drm->hdmi_device);
                 drm->hdmi_device = NULL;
                 return;
@@ -330,22 +352,24 @@ nouveau_drm_load(struct drm_device *dev, unsigned long flags)
         struct nouveau_drm *drm;
         int ret;
 
-        ret = nouveau_cli_create(pdev, "DRM", sizeof(*drm), (void**)&drm);
+        ret = nouveau_cli_create(nouveau_name(dev), "DRM", sizeof(*drm),
+                                 (void **)&drm);
         if (ret)
                 return ret;
 
         dev->dev_private = drm;
         drm->dev = dev;
+        nouveau_client(drm)->debug = nouveau_dbgopt(nouveau_debug, "DRM");
 
         INIT_LIST_HEAD(&drm->clients);
         spin_lock_init(&drm->tile.lock);
 
-        nouveau_get_hdmi_dev(dev);
+        nouveau_get_hdmi_dev(drm);
 
         /* make sure AGP controller is in a consistent state before we
          * (possibly) execute vbios init tables (see nouveau_agp.h)
          */
-        if (drm_pci_device_is_agp(dev) && dev->agp) {
+        if (pdev && drm_pci_device_is_agp(dev) && dev->agp) {
                 /* dummy device object, doesn't init anything, but allows
                  * agp code access to registers
                  */
@@ -486,13 +510,13 @@ nouveau_drm_remove(struct pci_dev *pdev)
 }
 
 static int
-nouveau_do_suspend(struct drm_device *dev)
+nouveau_do_suspend(struct drm_device *dev, bool runtime)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_cli *cli;
         int ret;
 
-        if (dev->mode_config.num_crtc) {
+        if (dev->mode_config.num_crtc && !runtime) {
                 NV_INFO(drm, "suspending display...\n");
                 ret = nouveau_display_suspend(dev);
                 if (ret)
@@ -566,7 +590,7 @@ int nouveau_pmops_suspend(struct device *dev)
         if (drm_dev->mode_config.num_crtc)
                 nouveau_fbcon_set_suspend(drm_dev, 1);
 
-        ret = nouveau_do_suspend(drm_dev);
+        ret = nouveau_do_suspend(drm_dev, false);
         if (ret)
                 return ret;
 
@@ -646,7 +670,7 @@ static int nouveau_pmops_freeze(struct device *dev)
         if (drm_dev->mode_config.num_crtc)
                 nouveau_fbcon_set_suspend(drm_dev, 1);
 
-        ret = nouveau_do_suspend(drm_dev);
+        ret = nouveau_do_suspend(drm_dev, false);
         return ret;
 }
 
@@ -671,7 +695,6 @@ static int nouveau_pmops_thaw(struct device *dev)
 static int
 nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv)
 {
-        struct pci_dev *pdev = dev->pdev;
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_cli *cli;
         char name[32], tmpname[TASK_COMM_LEN];
@@ -679,13 +702,15 @@ nouveau_drm_open(struct drm_device *dev, struct drm_file *fpriv)
 
         /* need to bring up power immediately if opening device */
         ret = pm_runtime_get_sync(dev->dev);
-        if (ret < 0)
+        if (ret < 0 && ret != -EACCES)
                 return ret;
 
         get_task_comm(tmpname, current);
         snprintf(name, sizeof(name), "%s[%d]", tmpname, pid_nr(fpriv->pid));
 
-        ret = nouveau_cli_create(pdev, name, sizeof(*cli), (void **)&cli);
+        ret = nouveau_cli_create(nouveau_name(dev), name, sizeof(*cli),
+                        (void **)&cli);
+
         if (ret)
                 goto out_suspend;
 
@@ -762,7 +787,7 @@ long nouveau_drm_ioctl(struct file *filp,
         dev = file_priv->minor->dev;
 
         ret = pm_runtime_get_sync(dev->dev);
-        if (ret < 0)
+        if (ret < 0 && ret != -EACCES)
                 return ret;
 
         ret = drm_ioctl(filp, cmd, arg);
@@ -882,7 +907,7 @@ static int nouveau_pmops_runtime_suspend(struct device *dev)
         drm_kms_helper_poll_disable(drm_dev);
         vga_switcheroo_set_dynamic_switch(pdev, VGA_SWITCHEROO_OFF);
         nouveau_switcheroo_optimus_dsm();
-        ret = nouveau_do_suspend(drm_dev);
+        ret = nouveau_do_suspend(drm_dev, true);
         pci_save_state(pdev);
         pci_disable_device(pdev);
         pci_set_power_state(pdev, PCI_D3cold);
@@ -908,8 +933,6 @@ static int nouveau_pmops_runtime_resume(struct device *dev)
         pci_set_master(pdev);
 
         ret = nouveau_do_resume(drm_dev);
-        if (drm_dev->mode_config.num_crtc)
-                nouveau_display_resume(drm_dev);
         drm_kms_helper_poll_enable(drm_dev);
         /* do magic */
         nv_mask(device, 0x88488, (1 << 25), (1 << 25));
@@ -980,6 +1003,25 @@ nouveau_drm_pci_driver = {
         .driver.pm = &nouveau_pm_ops,
 };
 
+int nouveau_drm_platform_probe(struct platform_device *pdev)
+{
+        struct nouveau_device *device;
+        int ret;
+
+        ret = nouveau_device_create(pdev, NOUVEAU_BUS_PLATFORM,
+                                    nouveau_platform_name(pdev),
+                                    dev_name(&pdev->dev), nouveau_config,
+                                    nouveau_debug, &device);
+
+        ret = drm_platform_init(&driver, pdev);
+        if (ret) {
+                nouveau_object_ref(NULL, (struct nouveau_object **)&device);
+                return ret;
+        }
+
+        return ret;
+}
+
 static int __init
 nouveau_drm_init(void)
 {
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.h b/drivers/gpu/drm/nouveau/nouveau_drm.h
index 23ca7a517246..7efbafaf7c1d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.h
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.h
@@ -161,10 +161,7 @@ int nouveau_pmops_resume(struct device *);
 #define NV_ERROR(cli, fmt, args...) nv_error((cli), fmt, ##args)
 #define NV_WARN(cli, fmt, args...) nv_warn((cli), fmt, ##args)
 #define NV_INFO(cli, fmt, args...) nv_info((cli), fmt, ##args)
-#define NV_DEBUG(cli, fmt, args...) do {                                       \
-        if (drm_debug & DRM_UT_DRIVER)                                         \
-                nv_info((cli), fmt, ##args);                                   \
-} while (0)
+#define NV_DEBUG(cli, fmt, args...) nv_debug((cli), fmt, ##args)
 
 extern int nouveau_modeset;
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_fbcon.c b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
index 7903e0ed3c75..64a42cfd3717 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_fbcon.c
@@ -528,10 +528,10 @@ nouveau_fbcon_set_suspend(struct drm_device *dev, int state)
         struct nouveau_drm *drm = nouveau_drm(dev);
         if (drm->fbcon) {
                 console_lock();
-                if (state == 0)
+                if (state == 1)
                         nouveau_fbcon_save_disable_accel(dev);
                 fb_set_suspend(drm->fbcon->helper.fbdev, state);
-                if (state == 1)
+                if (state == 0)
                         nouveau_fbcon_restore_accel(dev);
                 console_unlock();
         }
diff --git a/drivers/gpu/drm/nouveau/nouveau_gem.c b/drivers/gpu/drm/nouveau/nouveau_gem.c
index 27c3fd89e8ce..c90c0dc0afe8 100644
--- a/drivers/gpu/drm/nouveau/nouveau_gem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_gem.c
@@ -228,8 +228,6 @@ nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
         struct nouveau_bo *nvbo = NULL;
         int ret = 0;
 
-        drm->ttm.bdev.dev_mapping = drm->dev->dev_mapping;
-
         if (!pfb->memtype_valid(pfb, req->info.tile_flags)) {
                 NV_ERROR(cli, "bad page flags: 0x%08x\n", req->info.tile_flags);
                 return -EINVAL;
diff --git a/drivers/gpu/drm/nouveau/nouveau_hwmon.c b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
index 4aff04fa483c..19fd767bab10 100644
--- a/drivers/gpu/drm/nouveau/nouveau_hwmon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
@@ -383,8 +383,9 @@ nouveau_hwmon_set_pwm1_enable(struct device *d, struct device_attribute *a,
         long value;
         int ret;
 
-        if (strict_strtol(buf, 10, &value) == -EINVAL)
-                return -EINVAL;
+        ret = kstrtol(buf, 10, &value);
+        if (ret)
+                return ret;
 
         ret = therm->attr_set(therm, NOUVEAU_THERM_ATTR_FAN_MODE, value);
         if (ret)
@@ -587,18 +588,14 @@ nouveau_hwmon_init(struct drm_device *dev)
 
         /* set the default attributes */
         ret = sysfs_create_group(&hwmon_dev->kobj, &hwmon_default_attrgroup);
-        if (ret) {
-                if (ret)
-                        goto error;
-        }
+        if (ret)
+                goto error;
 
         /* if the card has a working thermal sensor */
         if (therm->temp_get(therm) >= 0) {
                 ret = sysfs_create_group(&hwmon_dev->kobj, &hwmon_temp_attrgroup);
-                if (ret) {
-                        if (ret)
-                                goto error;
-                }
+                if (ret)
+                        goto error;
         }
 
         /* if the card has a pwm fan */
diff --git a/drivers/gpu/drm/nouveau/nouveau_sysfs.c b/drivers/gpu/drm/nouveau/nouveau_sysfs.c
index 89201a17ce75..75dda2b07176 100644
--- a/drivers/gpu/drm/nouveau/nouveau_sysfs.c
+++ b/drivers/gpu/drm/nouveau/nouveau_sysfs.c
@@ -30,7 +30,7 @@
 static inline struct drm_device *
 drm_device(struct device *d)
 {
-        return pci_get_drvdata(to_pci_dev(d));
+        return dev_get_drvdata(d);
 }
 
 #define snappendf(p,r,f,a...) do {                                             \
@@ -132,9 +132,10 @@ nouveau_sysfs_fini(struct drm_device *dev)
 {
         struct nouveau_sysfs *sysfs = nouveau_sysfs(dev);
         struct nouveau_drm *drm = nouveau_drm(dev);
+        struct nouveau_device *device = nv_device(drm->device);
 
         if (sysfs->ctrl) {
-                device_remove_file(&dev->pdev->dev, &dev_attr_pstate);
+                device_remove_file(nv_device_base(device), &dev_attr_pstate);
                 nouveau_object_del(nv_object(drm), NVDRM_DEVICE, NVDRM_CONTROL);
         }
 
@@ -146,6 +147,7 @@ int
 nouveau_sysfs_init(struct drm_device *dev)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
+        struct nouveau_device *device = nv_device(drm->device);
         struct nouveau_sysfs *sysfs;
         int ret;
 
@@ -156,7 +158,7 @@ nouveau_sysfs_init(struct drm_device *dev)
         ret = nouveau_object_new(nv_object(drm), NVDRM_DEVICE, NVDRM_CONTROL,
                                  NV_CONTROL_CLASS, NULL, 0, &sysfs->ctrl);
         if (ret == 0)
-                device_create_file(&dev->pdev->dev, &dev_attr_pstate);
+                device_create_file(nv_device_base(device), &dev_attr_pstate);
 
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c
index d45d50da978f..ab0228f640a5 100644
--- a/drivers/gpu/drm/nouveau/nouveau_ttm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c
@@ -354,21 +354,26 @@ int
 nouveau_ttm_init(struct nouveau_drm *drm)
 {
         struct drm_device *dev = drm->dev;
+        struct nouveau_device *device = nv_device(drm->device);
         u32 bits;
         int ret;
 
         bits = nouveau_vmmgr(drm->device)->dma_bits;
-        if ( drm->agp.stat == ENABLED ||
-            !pci_dma_supported(dev->pdev, DMA_BIT_MASK(bits)))
-                bits = 32;
-
-        ret = pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(bits));
-        if (ret)
-                return ret;
-
-        ret = pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(bits));
-        if (ret)
-                pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(32));
+        if (nv_device_is_pci(device)) {
+                if (drm->agp.stat == ENABLED ||
+                     !pci_dma_supported(dev->pdev, DMA_BIT_MASK(bits)))
+                        bits = 32;
+
+                ret = pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(bits));
+                if (ret)
+                        return ret;
+
+                ret = pci_set_consistent_dma_mask(dev->pdev,
+                                                  DMA_BIT_MASK(bits));
+                if (ret)
+                        pci_set_consistent_dma_mask(dev->pdev,
+                                                    DMA_BIT_MASK(32));
+        }
 
         ret = nouveau_ttm_global_init(drm);
         if (ret)
@@ -376,7 +381,9 @@ nouveau_ttm_init(struct nouveau_drm *drm)
 
         ret = ttm_bo_device_init(&drm->ttm.bdev,
                                   drm->ttm.bo_global_ref.ref.object,
-                                  &nouveau_bo_driver, DRM_FILE_PAGE_OFFSET,
+                                  &nouveau_bo_driver,
+                                  dev->anon_inode->i_mapping,
+                                  DRM_FILE_PAGE_OFFSET,
                                   bits <= 32 ? true : false);
         if (ret) {
                 NV_ERROR(drm, "error initialising bo driver, %d\n", ret);
@@ -394,8 +401,8 @@ nouveau_ttm_init(struct nouveau_drm *drm)
                 return ret;
         }
 
-        drm->ttm.mtrr = arch_phys_wc_add(pci_resource_start(dev->pdev, 1),
-                                         pci_resource_len(dev->pdev, 1));
+        drm->ttm.mtrr = arch_phys_wc_add(nv_device_resource_start(device, 1),
+                                         nv_device_resource_len(device, 1));
 
         /* GART init */
         if (drm->agp.stat != ENABLED) {
diff --git a/drivers/gpu/drm/nouveau/nouveau_vga.c b/drivers/gpu/drm/nouveau/nouveau_vga.c
index 471347edc27e..fb84da3cb50d 100644
--- a/drivers/gpu/drm/nouveau/nouveau_vga.c
+++ b/drivers/gpu/drm/nouveau/nouveau_vga.c
@@ -84,6 +84,11 @@ nouveau_vga_init(struct nouveau_drm *drm)
 {
         struct drm_device *dev = drm->dev;
         bool runtime = false;
+
+        /* only relevant for PCI devices */
+        if (!dev->pdev)
+                return;
+
         vga_client_register(dev->pdev, dev, NULL, nouveau_vga_set_decode);
 
         if (nouveau_runtime_pm == 1)
diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c
index 2dccafc6e9db..58af547b0b93 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.c
+++ b/drivers/gpu/drm/nouveau/nv50_display.c
@@ -651,7 +651,7 @@ nv50_crtc_set_dither(struct nouveau_crtc *nv_crtc, bool update)
         nv_connector = nouveau_crtc_connector_get(nv_crtc);
         connector = &nv_connector->base;
         if (nv_connector->dithering_mode == DITHERING_MODE_AUTO) {
-                if (nv_crtc->base.fb->depth > connector->display_info.bpc * 3)
+                if (nv_crtc->base.primary->fb->depth > connector->display_info.bpc * 3)
                         mode = DITHERING_MODE_DYNAMIC2X2;
         } else {
                 mode = nv_connector->dithering_mode;
@@ -785,7 +785,8 @@ nv50_crtc_set_scale(struct nouveau_crtc *nv_crtc, bool update)
 
                 if (update) {
                         nv50_display_flip_stop(crtc);
-                        nv50_display_flip_next(crtc, crtc->fb, NULL, 1);
+                        nv50_display_flip_next(crtc, crtc->primary->fb,
+                                               NULL, 1);
                 }
         }
 
@@ -1028,7 +1029,7 @@ nv50_crtc_commit(struct drm_crtc *crtc)
         }
 
         nv50_crtc_cursor_show_hide(nv_crtc, nv_crtc->cursor.visible, true);
-        nv50_display_flip_next(crtc, crtc->fb, NULL, 1);
+        nv50_display_flip_next(crtc, crtc->primary->fb, NULL, 1);
 }
 
 static bool
@@ -1042,7 +1043,7 @@ nv50_crtc_mode_fixup(struct drm_crtc *crtc, const struct drm_display_mode *mode,
 static int
 nv50_crtc_swap_fbs(struct drm_crtc *crtc, struct drm_framebuffer *old_fb)
 {
-        struct nouveau_framebuffer *nvfb = nouveau_framebuffer(crtc->fb);
+        struct nouveau_framebuffer *nvfb = nouveau_framebuffer(crtc->primary->fb);
         struct nv50_head *head = nv50_head(crtc);
         int ret;
 
@@ -1139,7 +1140,7 @@ nv50_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *umode,
         nv50_crtc_set_dither(nv_crtc, false);
         nv50_crtc_set_scale(nv_crtc, false);
         nv50_crtc_set_color_vibrance(nv_crtc, false);
-        nv50_crtc_set_image(nv_crtc, crtc->fb, x, y, false);
+        nv50_crtc_set_image(nv_crtc, crtc->primary->fb, x, y, false);
         return 0;
 }
 
@@ -1151,7 +1152,7 @@ nv50_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
         int ret;
 
-        if (!crtc->fb) {
+        if (!crtc->primary->fb) {
                 NV_DEBUG(drm, "No FB bound\n");
                 return 0;
         }
@@ -1161,8 +1162,8 @@ nv50_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
                 return ret;
 
         nv50_display_flip_stop(crtc);
-        nv50_crtc_set_image(nv_crtc, crtc->fb, x, y, true);
-        nv50_display_flip_next(crtc, crtc->fb, NULL, 1);
+        nv50_crtc_set_image(nv_crtc, crtc->primary->fb, x, y, true);
+        nv50_display_flip_next(crtc, crtc->primary->fb, NULL, 1);
         return 0;
 }
 
diff --git a/drivers/gpu/drm/omapdrm/omap_crtc.c b/drivers/gpu/drm/omapdrm/omap_crtc.c
index 4313bb0a49a6..355157e4f78d 100644
--- a/drivers/gpu/drm/omapdrm/omap_crtc.c
+++ b/drivers/gpu/drm/omapdrm/omap_crtc.c
@@ -245,7 +245,7 @@ static int omap_crtc_mode_set(struct drm_crtc *crtc,
         copy_timings_drm_to_omap(&omap_crtc->timings, mode);
         omap_crtc->full_update = true;
 
-        return omap_plane_mode_set(omap_crtc->plane, crtc, crtc->fb,
+        return omap_plane_mode_set(omap_crtc->plane, crtc, crtc->primary->fb,
                         0, 0, mode->hdisplay, mode->vdisplay,
                         x << 16, y << 16,
                         mode->hdisplay << 16, mode->vdisplay << 16,
@@ -273,7 +273,7 @@ static int omap_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
         struct drm_plane *plane = omap_crtc->plane;
         struct drm_display_mode *mode = &crtc->mode;
 
-        return omap_plane_mode_set(plane, crtc, crtc->fb,
+        return omap_plane_mode_set(plane, crtc, crtc->primary->fb,
                         0, 0, mode->hdisplay, mode->vdisplay,
                         x << 16, y << 16,
                         mode->hdisplay << 16, mode->vdisplay << 16,
@@ -308,14 +308,14 @@ static void page_flip_worker(struct work_struct *work)
         struct drm_gem_object *bo;
 
         mutex_lock(&crtc->mutex);
-        omap_plane_mode_set(omap_crtc->plane, crtc, crtc->fb,
+        omap_plane_mode_set(omap_crtc->plane, crtc, crtc->primary->fb,
                         0, 0, mode->hdisplay, mode->vdisplay,
                         crtc->x << 16, crtc->y << 16,
                         mode->hdisplay << 16, mode->vdisplay << 16,
                         vblank_cb, crtc);
         mutex_unlock(&crtc->mutex);
 
-        bo = omap_framebuffer_bo(crtc->fb, 0);
+        bo = omap_framebuffer_bo(crtc->primary->fb, 0);
         drm_gem_object_unreference_unlocked(bo);
 }
 
@@ -336,9 +336,10 @@ static int omap_crtc_page_flip_locked(struct drm_crtc *crtc,
 {
         struct drm_device *dev = crtc->dev;
         struct omap_crtc *omap_crtc = to_omap_crtc(crtc);
+        struct drm_plane *primary = crtc->primary;
         struct drm_gem_object *bo;
 
-        DBG("%d -> %d (event=%p)", crtc->fb ? crtc->fb->base.id : -1,
+        DBG("%d -> %d (event=%p)", primary->fb ? primary->fb->base.id : -1,
                         fb->base.id, event);
 
         if (omap_crtc->old_fb) {
@@ -347,7 +348,7 @@ static int omap_crtc_page_flip_locked(struct drm_crtc *crtc,
         }
 
         omap_crtc->event = event;
-        crtc->fb = fb;
+        primary->fb = fb;
 
         /*
          * Hold a reference temporarily until the crtc is updated
diff --git a/drivers/gpu/drm/omapdrm/omap_fb.c b/drivers/gpu/drm/omapdrm/omap_fb.c
index f466c4aaee94..d2b8c49bfb4a 100644
--- a/drivers/gpu/drm/omapdrm/omap_fb.c
+++ b/drivers/gpu/drm/omapdrm/omap_fb.c
@@ -306,13 +306,14 @@ struct drm_connector *omap_framebuffer_get_next_connector(
         struct drm_connector *connector = from;
 
         if (!from)
-                return list_first_entry(connector_list, typeof(*from), head);
+                return list_first_entry_or_null(connector_list, typeof(*from),
+                                                head);
 
         list_for_each_entry_from(connector, connector_list, head) {
                 if (connector != from) {
                         struct drm_encoder *encoder = connector->encoder;
                         struct drm_crtc *crtc = encoder ? encoder->crtc : NULL;
-                        if (crtc && crtc->fb == fb)
+                        if (crtc && crtc->primary->fb == fb)
                                 return connector;
 
                 }
diff --git a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c
index 5aec3e81fe24..c8d972763889 100644
--- a/drivers/gpu/drm/omapdrm/omap_gem.c
+++ b/drivers/gpu/drm/omapdrm/omap_gem.c
@@ -153,24 +153,24 @@ static struct {
 static void evict_entry(struct drm_gem_object *obj,
                 enum tiler_fmt fmt, struct usergart_entry *entry)
 {
-        if (obj->dev->dev_mapping) {
-                struct omap_gem_object *omap_obj = to_omap_bo(obj);
-                int n = usergart[fmt].height;
-                size_t size = PAGE_SIZE * n;
-                loff_t off = mmap_offset(obj) +
-                                (entry->obj_pgoff << PAGE_SHIFT);
-                const int m = 1 + ((omap_obj->width << fmt) / PAGE_SIZE);
-                if (m > 1) {
-                        int i;
-                        /* if stride > than PAGE_SIZE then sparse mapping: */
-                        for (i = n; i > 0; i--) {
-                                unmap_mapping_range(obj->dev->dev_mapping,
-                                                off, PAGE_SIZE, 1);
-                                off += PAGE_SIZE * m;
-                        }
-                } else {
-                        unmap_mapping_range(obj->dev->dev_mapping, off, size, 1);
+        struct omap_gem_object *omap_obj = to_omap_bo(obj);
+        int n = usergart[fmt].height;
+        size_t size = PAGE_SIZE * n;
+        loff_t off = mmap_offset(obj) +
+                        (entry->obj_pgoff << PAGE_SHIFT);
+        const int m = 1 + ((omap_obj->width << fmt) / PAGE_SIZE);
+
+        if (m > 1) {
+                int i;
+                /* if stride > than PAGE_SIZE then sparse mapping: */
+                for (i = n; i > 0; i--) {
+                        unmap_mapping_range(obj->dev->anon_inode->i_mapping,
+                                            off, PAGE_SIZE, 1);
+                        off += PAGE_SIZE * m;
                 }
+        } else {
+                unmap_mapping_range(obj->dev->anon_inode->i_mapping,
+                                    off, size, 1);
         }
 
         entry->obj = NULL;
diff --git a/drivers/gpu/drm/panel/Kconfig b/drivers/gpu/drm/panel/Kconfig
index 3e0f13d1bc84..4ec874da5668 100644
--- a/drivers/gpu/drm/panel/Kconfig
+++ b/drivers/gpu/drm/panel/Kconfig
@@ -16,4 +16,18 @@ config DRM_PANEL_SIMPLE
           that it can be automatically turned off when the panel goes into a
           low power state.
 
+config DRM_PANEL_LD9040
+        tristate "LD9040 RGB/SPI panel"
+        depends on DRM && DRM_PANEL
+        depends on OF
+        select SPI
+        select VIDEOMODE_HELPERS
+
+config DRM_PANEL_S6E8AA0
+        tristate "S6E8AA0 DSI video mode panel"
+        depends on DRM && DRM_PANEL
+        depends on OF
+        select DRM_MIPI_DSI
+        select VIDEOMODE_HELPERS
+
 endmenu
diff --git a/drivers/gpu/drm/panel/Makefile b/drivers/gpu/drm/panel/Makefile
index af9dfa235b94..8b929212fad7 100644
--- a/drivers/gpu/drm/panel/Makefile
+++ b/drivers/gpu/drm/panel/Makefile
@@ -1 +1,3 @@
 obj-$(CONFIG_DRM_PANEL_SIMPLE) += panel-simple.o
+obj-$(CONFIG_DRM_PANEL_LD9040) += panel-ld9040.o
+obj-$(CONFIG_DRM_PANEL_S6E8AA0) += panel-s6e8aa0.o
diff --git a/drivers/gpu/drm/panel/panel-ld9040.c b/drivers/gpu/drm/panel/panel-ld9040.c
new file mode 100644
index 000000000000..1f1f8371a199
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-ld9040.c
@@ -0,0 +1,376 @@
+/*
+ * ld9040 AMOLED LCD drm_panel driver.
+ *
+ * Copyright (c) 2014 Samsung Electronics Co., Ltd
+ * Derived from drivers/video/backlight/ld9040.c
+ *
+ * Andrzej Hajda <a.hajda@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+*/
+
+#include <drm/drmP.h>
+#include <drm/drm_panel.h>
+
+#include <linux/gpio/consumer.h>
+#include <linux/regulator/consumer.h>
+#include <linux/spi/spi.h>
+
+#include <video/mipi_display.h>
+#include <video/of_videomode.h>
+#include <video/videomode.h>
+
+/* Manufacturer Command Set */
+#define MCS_MANPWR              0xb0
+#define MCS_ELVSS_ON            0xb1
+#define MCS_USER_SETTING        0xf0
+#define MCS_DISPCTL             0xf2
+#define MCS_GTCON               0xf7
+#define MCS_PANEL_CONDITION     0xf8
+#define MCS_GAMMA_SET1          0xf9
+#define MCS_GAMMA_CTRL          0xfb
+
+/* array of gamma tables for gamma value 2.2 */
+static u8 const ld9040_gammas[25][22] = {
+        { 0xf9, 0x00, 0x13, 0xb2, 0xba, 0xd2, 0x00, 0x30, 0x00, 0xaf, 0xc0,
+          0xb8, 0xcd, 0x00, 0x3d, 0x00, 0xa8, 0xb8, 0xb7, 0xcd, 0x00, 0x44 },
+        { 0xf9, 0x00, 0x13, 0xb9, 0xb9, 0xd0, 0x00, 0x3c, 0x00, 0xaf, 0xbf,
+          0xb6, 0xcb, 0x00, 0x4b, 0x00, 0xa8, 0xb9, 0xb5, 0xcc, 0x00, 0x52 },
+        { 0xf9, 0x00, 0x13, 0xba, 0xb9, 0xcd, 0x00, 0x41, 0x00, 0xb0, 0xbe,
+          0xb5, 0xc9, 0x00, 0x51, 0x00, 0xa9, 0xb9, 0xb5, 0xca, 0x00, 0x57 },
+        { 0xf9, 0x00, 0x13, 0xb9, 0xb8, 0xcd, 0x00, 0x46, 0x00, 0xb1, 0xbc,
+          0xb5, 0xc8, 0x00, 0x56, 0x00, 0xaa, 0xb8, 0xb4, 0xc9, 0x00, 0x5d },
+        { 0xf9, 0x00, 0x13, 0xba, 0xb8, 0xcb, 0x00, 0x4b, 0x00, 0xb3, 0xbc,
+          0xb4, 0xc7, 0x00, 0x5c, 0x00, 0xac, 0xb8, 0xb4, 0xc8, 0x00, 0x62 },
+        { 0xf9, 0x00, 0x13, 0xbb, 0xb7, 0xca, 0x00, 0x4f, 0x00, 0xb4, 0xbb,
+          0xb3, 0xc7, 0x00, 0x60, 0x00, 0xad, 0xb8, 0xb4, 0xc7, 0x00, 0x67 },
+        { 0xf9, 0x00, 0x47, 0xba, 0xb6, 0xca, 0x00, 0x53, 0x00, 0xb5, 0xbb,
+          0xb3, 0xc6, 0x00, 0x65, 0x00, 0xae, 0xb8, 0xb3, 0xc7, 0x00, 0x6c },
+        { 0xf9, 0x00, 0x71, 0xbb, 0xb5, 0xc8, 0x00, 0x57, 0x00, 0xb5, 0xbb,
+          0xb0, 0xc5, 0x00, 0x6a, 0x00, 0xae, 0xb9, 0xb1, 0xc6, 0x00, 0x70 },
+        { 0xf9, 0x00, 0x7b, 0xbb, 0xb4, 0xc8, 0x00, 0x5b, 0x00, 0xb5, 0xba,
+          0xb1, 0xc4, 0x00, 0x6e, 0x00, 0xae, 0xb9, 0xb0, 0xc5, 0x00, 0x75 },
+        { 0xf9, 0x00, 0x82, 0xba, 0xb4, 0xc7, 0x00, 0x5f, 0x00, 0xb5, 0xba,
+          0xb0, 0xc3, 0x00, 0x72, 0x00, 0xae, 0xb8, 0xb0, 0xc3, 0x00, 0x7a },
+        { 0xf9, 0x00, 0x89, 0xba, 0xb3, 0xc8, 0x00, 0x62, 0x00, 0xb6, 0xba,
+          0xaf, 0xc3, 0x00, 0x76, 0x00, 0xaf, 0xb7, 0xae, 0xc4, 0x00, 0x7e },
+        { 0xf9, 0x00, 0x8b, 0xb9, 0xb3, 0xc7, 0x00, 0x65, 0x00, 0xb7, 0xb8,
+          0xaf, 0xc3, 0x00, 0x7a, 0x00, 0x80, 0xb6, 0xae, 0xc4, 0x00, 0x81 },
+        { 0xf9, 0x00, 0x93, 0xba, 0xb3, 0xc5, 0x00, 0x69, 0x00, 0xb8, 0xb9,
+          0xae, 0xc1, 0x00, 0x7f, 0x00, 0xb0, 0xb6, 0xae, 0xc3, 0x00, 0x85 },
+        { 0xf9, 0x00, 0x97, 0xba, 0xb2, 0xc5, 0x00, 0x6c, 0x00, 0xb8, 0xb8,
+          0xae, 0xc1, 0x00, 0x82, 0x00, 0xb0, 0xb6, 0xae, 0xc2, 0x00, 0x89 },
+        { 0xf9, 0x00, 0x9a, 0xba, 0xb1, 0xc4, 0x00, 0x6f, 0x00, 0xb8, 0xb8,
+          0xad, 0xc0, 0x00, 0x86, 0x00, 0xb0, 0xb7, 0xad, 0xc0, 0x00, 0x8d },
+        { 0xf9, 0x00, 0x9c, 0xb9, 0xb0, 0xc4, 0x00, 0x72, 0x00, 0xb8, 0xb8,
+          0xac, 0xbf, 0x00, 0x8a, 0x00, 0xb0, 0xb6, 0xac, 0xc0, 0x00, 0x91 },
+        { 0xf9, 0x00, 0x9e, 0xba, 0xb0, 0xc2, 0x00, 0x75, 0x00, 0xb9, 0xb8,
+          0xab, 0xbe, 0x00, 0x8e, 0x00, 0xb0, 0xb6, 0xac, 0xbf, 0x00, 0x94 },
+        { 0xf9, 0x00, 0xa0, 0xb9, 0xaf, 0xc3, 0x00, 0x77, 0x00, 0xb9, 0xb7,
+          0xab, 0xbe, 0x00, 0x90, 0x00, 0xb0, 0xb6, 0xab, 0xbf, 0x00, 0x97 },
+        { 0xf9, 0x00, 0xa2, 0xb9, 0xaf, 0xc2, 0x00, 0x7a, 0x00, 0xb9, 0xb7,
+          0xaa, 0xbd, 0x00, 0x94, 0x00, 0xb0, 0xb5, 0xab, 0xbf, 0x00, 0x9a },
+        { 0xf9, 0x00, 0xa4, 0xb9, 0xaf, 0xc1, 0x00, 0x7d, 0x00, 0xb9, 0xb6,
+          0xaa, 0xbb, 0x00, 0x97, 0x00, 0xb1, 0xb5, 0xaa, 0xbf, 0x00, 0x9d },
+        { 0xf9, 0x00, 0xa4, 0xb8, 0xb0, 0xbf, 0x00, 0x80, 0x00, 0xb8, 0xb6,
+          0xaa, 0xbc, 0x00, 0x9a, 0x00, 0xb0, 0xb5, 0xab, 0xbd, 0x00, 0xa0 },
+        { 0xf9, 0x00, 0xa8, 0xb8, 0xae, 0xbe, 0x00, 0x84, 0x00, 0xb9, 0xb7,
+          0xa8, 0xbc, 0x00, 0x9d, 0x00, 0xb2, 0xb5, 0xaa, 0xbc, 0x00, 0xa4 },
+        { 0xf9, 0x00, 0xa9, 0xb6, 0xad, 0xbf, 0x00, 0x86, 0x00, 0xb8, 0xb5,
+          0xa8, 0xbc, 0x00, 0xa0, 0x00, 0xb3, 0xb3, 0xa9, 0xbc, 0x00, 0xa7 },
+        { 0xf9, 0x00, 0xa9, 0xb7, 0xae, 0xbd, 0x00, 0x89, 0x00, 0xb7, 0xb6,
+          0xa8, 0xba, 0x00, 0xa4, 0x00, 0xb1, 0xb4, 0xaa, 0xbb, 0x00, 0xaa },
+        { 0xf9, 0x00, 0xa7, 0xb4, 0xae, 0xbf, 0x00, 0x91, 0x00, 0xb2, 0xb4,
+          0xaa, 0xbb, 0x00, 0xac, 0x00, 0xb3, 0xb1, 0xaa, 0xbc, 0x00, 0xb3 },
+};
+
+struct ld9040 {
+        struct device *dev;
+        struct drm_panel panel;
+
+        struct regulator_bulk_data supplies[2];
+        struct gpio_desc *reset_gpio;
+        u32 power_on_delay;
+        u32 reset_delay;
+        struct videomode vm;
+        u32 width_mm;
+        u32 height_mm;
+
+        int brightness;
+
+        /* This field is tested by functions directly accessing bus before
+         * transfer, transfer is skipped if it is set. In case of transfer
+         * failure or unexpected response the field is set to error value.
+         * Such construct allows to eliminate many checks in higher level
+         * functions.
+         */
+        int error;
+};
+
+#define panel_to_ld9040(p) container_of(p, struct ld9040, panel)
+
+static int ld9040_clear_error(struct ld9040 *ctx)
+{
+        int ret = ctx->error;
+
+        ctx->error = 0;
+        return ret;
+}
+
+static int ld9040_spi_write_word(struct ld9040 *ctx, u16 data)
+{
+        struct spi_device *spi = to_spi_device(ctx->dev);
+        struct spi_transfer xfer = {
+                .len            = 2,
+                .tx_buf         = &data,
+        };
+        struct spi_message msg;
+
+        spi_message_init(&msg);
+        spi_message_add_tail(&xfer, &msg);
+
+        return spi_sync(spi, &msg);
+}
+
+static void ld9040_dcs_write(struct ld9040 *ctx, const u8 *data, size_t len)
+{
+        int ret = 0;
+
+        if (ctx->error < 0 || len == 0)
+                return;
+
+        dev_dbg(ctx->dev, "writing dcs seq: %*ph\n", len, data);
+        ret = ld9040_spi_write_word(ctx, *data);
+
+        while (!ret && --len) {
+                ++data;
+                ret = ld9040_spi_write_word(ctx, *data | 0x100);
+        }
+
+        if (ret) {
+                dev_err(ctx->dev, "error %d writing dcs seq: %*ph\n", ret, len,
+                        data);
+                ctx->error = ret;
+        }
+
+        usleep_range(300, 310);
+}
+
+#define ld9040_dcs_write_seq_static(ctx, seq...) \
+({\
+        static const u8 d[] = { seq };\
+        ld9040_dcs_write(ctx, d, ARRAY_SIZE(d));\
+})
+
+static void ld9040_brightness_set(struct ld9040 *ctx)
+{
+        ld9040_dcs_write(ctx, ld9040_gammas[ctx->brightness],
+                         ARRAY_SIZE(ld9040_gammas[ctx->brightness]));
+
+        ld9040_dcs_write_seq_static(ctx, MCS_GAMMA_CTRL, 0x02, 0x5a);
+}
+
+static void ld9040_init(struct ld9040 *ctx)
+{
+        ld9040_dcs_write_seq_static(ctx, MCS_USER_SETTING, 0x5a, 0x5a);
+        ld9040_dcs_write_seq_static(ctx, MCS_PANEL_CONDITION,
+                0x05, 0x65, 0x96, 0x71, 0x7d, 0x19, 0x3b, 0x0d,
+                0x19, 0x7e, 0x0d, 0xe2, 0x00, 0x00, 0x7e, 0x7d,
+                0x07, 0x07, 0x20, 0x20, 0x20, 0x02, 0x02);
+        ld9040_dcs_write_seq_static(ctx, MCS_DISPCTL,
+                0x02, 0x08, 0x08, 0x10, 0x10);
+        ld9040_dcs_write_seq_static(ctx, MCS_MANPWR, 0x04);
+        ld9040_dcs_write_seq_static(ctx, MCS_ELVSS_ON, 0x0d, 0x00, 0x16);
+        ld9040_dcs_write_seq_static(ctx, MCS_GTCON, 0x09, 0x00, 0x00);
+        ld9040_brightness_set(ctx);
+        ld9040_dcs_write_seq_static(ctx, MIPI_DCS_EXIT_SLEEP_MODE);
+        ld9040_dcs_write_seq_static(ctx, MIPI_DCS_SET_DISPLAY_ON);
+}
+
+static int ld9040_power_on(struct ld9040 *ctx)
+{
+        int ret;
+
+        ret = regulator_bulk_enable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+        if (ret < 0)
+                return ret;
+
+        msleep(ctx->power_on_delay);
+        gpiod_set_value(ctx->reset_gpio, 0);
+        msleep(ctx->reset_delay);
+        gpiod_set_value(ctx->reset_gpio, 1);
+        msleep(ctx->reset_delay);
+
+        return 0;
+}
+
+static int ld9040_power_off(struct ld9040 *ctx)
+{
+        return regulator_bulk_disable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+}
+
+static int ld9040_disable(struct drm_panel *panel)
+{
+        struct ld9040 *ctx = panel_to_ld9040(panel);
+
+        msleep(120);
+        ld9040_dcs_write_seq_static(ctx, MIPI_DCS_SET_DISPLAY_OFF);
+        ld9040_dcs_write_seq_static(ctx, MIPI_DCS_ENTER_SLEEP_MODE);
+        msleep(40);
+
+        ld9040_clear_error(ctx);
+
+        return ld9040_power_off(ctx);
+}
+
+static int ld9040_enable(struct drm_panel *panel)
+{
+        struct ld9040 *ctx = panel_to_ld9040(panel);
+        int ret;
+
+        ret = ld9040_power_on(ctx);
+        if (ret < 0)
+                return ret;
+
+        ld9040_init(ctx);
+
+        ret = ld9040_clear_error(ctx);
+
+        if (ret < 0)
+                ld9040_disable(panel);
+
+        return ret;
+}
+
+static int ld9040_get_modes(struct drm_panel *panel)
+{
+        struct drm_connector *connector = panel->connector;
+        struct ld9040 *ctx = panel_to_ld9040(panel);
+        struct drm_display_mode *mode;
+
+        mode = drm_mode_create(connector->dev);
+        if (!mode) {
+                DRM_ERROR("failed to create a new display mode\n");
+                return 0;
+        }
+
+        drm_display_mode_from_videomode(&ctx->vm, mode);
+        mode->width_mm = ctx->width_mm;
+        mode->height_mm = ctx->height_mm;
+        connector->display_info.width_mm = mode->width_mm;
+        connector->display_info.height_mm = mode->height_mm;
+
+        mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+        drm_mode_probed_add(connector, mode);
+
+        return 1;
+}
+
+static const struct drm_panel_funcs ld9040_drm_funcs = {
+        .disable = ld9040_disable,
+        .enable = ld9040_enable,
+        .get_modes = ld9040_get_modes,
+};
+
+static int ld9040_parse_dt(struct ld9040 *ctx)
+{
+        struct device *dev = ctx->dev;
+        struct device_node *np = dev->of_node;
+        int ret;
+
+        ret = of_get_videomode(np, &ctx->vm, 0);
+        if (ret < 0)
+                return ret;
+
+        of_property_read_u32(np, "power-on-delay", &ctx->power_on_delay);
+        of_property_read_u32(np, "reset-delay", &ctx->reset_delay);
+        of_property_read_u32(np, "panel-width-mm", &ctx->width_mm);
+        of_property_read_u32(np, "panel-height-mm", &ctx->height_mm);
+
+        return 0;
+}
+
+static int ld9040_probe(struct spi_device *spi)
+{
+        struct device *dev = &spi->dev;
+        struct ld9040 *ctx;
+        int ret;
+
+        ctx = devm_kzalloc(dev, sizeof(struct ld9040), GFP_KERNEL);
+        if (!ctx)
+                return -ENOMEM;
+
+        spi_set_drvdata(spi, ctx);
+
+        ctx->dev = dev;
+        ctx->brightness = ARRAY_SIZE(ld9040_gammas) - 1;
+
+        ret = ld9040_parse_dt(ctx);
+        if (ret < 0)
+                return ret;
+
+        ctx->supplies[0].supply = "vdd3";
+        ctx->supplies[1].supply = "vci";
+        ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(ctx->supplies),
+                                      ctx->supplies);
+        if (ret < 0)
+                return ret;
+
+        ctx->reset_gpio = devm_gpiod_get(dev, "reset");
+        if (IS_ERR(ctx->reset_gpio)) {
+                dev_err(dev, "cannot get reset-gpios %ld\n",
+                        PTR_ERR(ctx->reset_gpio));
+                return PTR_ERR(ctx->reset_gpio);
+        }
+        ret = gpiod_direction_output(ctx->reset_gpio, 1);
+        if (ret < 0) {
+                dev_err(dev, "cannot configure reset-gpios %d\n", ret);
+                return ret;
+        }
+
+        spi->bits_per_word = 9;
+        ret = spi_setup(spi);
+        if (ret < 0) {
+                dev_err(dev, "spi setup failed.\n");
+                return ret;
+        }
+
+        drm_panel_init(&ctx->panel);
+        ctx->panel.dev = dev;
+        ctx->panel.funcs = &ld9040_drm_funcs;
+
+        return drm_panel_add(&ctx->panel);
+}
+
+static int ld9040_remove(struct spi_device *spi)
+{
+        struct ld9040 *ctx = spi_get_drvdata(spi);
+
+        ld9040_power_off(ctx);
+        drm_panel_remove(&ctx->panel);
+
+        return 0;
+}
+
+static struct of_device_id ld9040_of_match[] = {
+        { .compatible = "samsung,ld9040" },
+        { }
+};
+MODULE_DEVICE_TABLE(of, ld9040_of_match);
+
+static struct spi_driver ld9040_driver = {
+        .probe          = ld9040_probe,
+        .remove         = ld9040_remove,
+        .driver = {
+                .name   = "ld9040",
+                .owner  = THIS_MODULE,
+                .of_match_table = ld9040_of_match,
+        },
+};
+module_spi_driver(ld9040_driver);
+
+MODULE_AUTHOR("Andrzej Hajda <a.hajda@samsung.com>");
+MODULE_DESCRIPTION("ld9040 LCD Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/panel/panel-s6e8aa0.c b/drivers/gpu/drm/panel/panel-s6e8aa0.c
new file mode 100644
index 000000000000..35941d2412b8
--- /dev/null
+++ b/drivers/gpu/drm/panel/panel-s6e8aa0.c
@@ -0,0 +1,1069 @@
+/*
+ * MIPI-DSI based s6e8aa0 AMOLED LCD 5.3 inch panel driver.
+ *
+ * Copyright (c) 2013 Samsung Electronics Co., Ltd
+ *
+ * Inki Dae, <inki.dae@samsung.com>
+ * Donghwa Lee, <dh09.lee@samsung.com>
+ * Joongmock Shin <jmock.shin@samsung.com>
+ * Eunchul Kim <chulspro.kim@samsung.com>
+ * Tomasz Figa <t.figa@samsung.com>
+ * Andrzej Hajda <a.hajda@samsung.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+*/
+
+#include <drm/drmP.h>
+#include <drm/drm_mipi_dsi.h>
+#include <drm/drm_panel.h>
+
+#include <linux/gpio/consumer.h>
+#include <linux/regulator/consumer.h>
+
+#include <video/mipi_display.h>
+#include <video/of_videomode.h>
+#include <video/videomode.h>
+
+#define LDI_MTP_LENGTH                  24
+#define GAMMA_LEVEL_NUM                 25
+#define GAMMA_TABLE_LEN                 26
+
+#define PANELCTL_SS_MASK                (1 << 5)
+#define PANELCTL_SS_1_800               (0 << 5)
+#define PANELCTL_SS_800_1               (1 << 5)
+#define PANELCTL_GTCON_MASK             (7 << 2)
+#define PANELCTL_GTCON_110              (6 << 2)
+#define PANELCTL_GTCON_111              (7 << 2)
+
+#define PANELCTL_CLK1_CON_MASK          (7 << 3)
+#define PANELCTL_CLK1_000               (0 << 3)
+#define PANELCTL_CLK1_001               (1 << 3)
+#define PANELCTL_CLK2_CON_MASK          (7 << 0)
+#define PANELCTL_CLK2_000               (0 << 0)
+#define PANELCTL_CLK2_001               (1 << 0)
+
+#define PANELCTL_INT1_CON_MASK          (7 << 3)
+#define PANELCTL_INT1_000               (0 << 3)
+#define PANELCTL_INT1_001               (1 << 3)
+#define PANELCTL_INT2_CON_MASK          (7 << 0)
+#define PANELCTL_INT2_000               (0 << 0)
+#define PANELCTL_INT2_001               (1 << 0)
+
+#define PANELCTL_BICTL_CON_MASK         (7 << 3)
+#define PANELCTL_BICTL_000              (0 << 3)
+#define PANELCTL_BICTL_001              (1 << 3)
+#define PANELCTL_BICTLB_CON_MASK        (7 << 0)
+#define PANELCTL_BICTLB_000             (0 << 0)
+#define PANELCTL_BICTLB_001             (1 << 0)
+
+#define PANELCTL_EM_CLK1_CON_MASK       (7 << 3)
+#define PANELCTL_EM_CLK1_110            (6 << 3)
+#define PANELCTL_EM_CLK1_111            (7 << 3)
+#define PANELCTL_EM_CLK1B_CON_MASK      (7 << 0)
+#define PANELCTL_EM_CLK1B_110           (6 << 0)
+#define PANELCTL_EM_CLK1B_111           (7 << 0)
+
+#define PANELCTL_EM_CLK2_CON_MASK       (7 << 3)
+#define PANELCTL_EM_CLK2_110            (6 << 3)
+#define PANELCTL_EM_CLK2_111            (7 << 3)
+#define PANELCTL_EM_CLK2B_CON_MASK      (7 << 0)
+#define PANELCTL_EM_CLK2B_110           (6 << 0)
+#define PANELCTL_EM_CLK2B_111           (7 << 0)
+
+#define PANELCTL_EM_INT1_CON_MASK       (7 << 3)
+#define PANELCTL_EM_INT1_000            (0 << 3)
+#define PANELCTL_EM_INT1_001            (1 << 3)
+#define PANELCTL_EM_INT2_CON_MASK       (7 << 0)
+#define PANELCTL_EM_INT2_000            (0 << 0)
+#define PANELCTL_EM_INT2_001            (1 << 0)
+
+#define AID_DISABLE                     (0x4)
+#define AID_1                           (0x5)
+#define AID_2                           (0x6)
+#define AID_3                           (0x7)
+
+typedef u8 s6e8aa0_gamma_table[GAMMA_TABLE_LEN];
+
+struct s6e8aa0_variant {
+        u8 version;
+        const s6e8aa0_gamma_table *gamma_tables;
+};
+
+struct s6e8aa0 {
+        struct device *dev;
+        struct drm_panel panel;
+
+        struct regulator_bulk_data supplies[2];
+        struct gpio_desc *reset_gpio;
+        u32 power_on_delay;
+        u32 reset_delay;
+        u32 init_delay;
+        bool flip_horizontal;
+        bool flip_vertical;
+        struct videomode vm;
+        u32 width_mm;
+        u32 height_mm;
+
+        u8 version;
+        u8 id;
+        const struct s6e8aa0_variant *variant;
+        int brightness;
+
+        /* This field is tested by functions directly accessing DSI bus before
+         * transfer, transfer is skipped if it is set. In case of transfer
+         * failure or unexpected response the field is set to error value.
+         * Such construct allows to eliminate many checks in higher level
+         * functions.
+         */
+        int error;
+};
+
+#define panel_to_s6e8aa0(p) container_of(p, struct s6e8aa0, panel)
+
+static int s6e8aa0_clear_error(struct s6e8aa0 *ctx)
+{
+        int ret = ctx->error;
+
+        ctx->error = 0;
+        return ret;
+}
+
+static void s6e8aa0_dcs_write(struct s6e8aa0 *ctx, const void *data, size_t len)
+{
+        struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
+        int ret;
+
+        if (ctx->error < 0)
+                return;
+
+        ret = mipi_dsi_dcs_write(dsi, dsi->channel, data, len);
+        if (ret < 0) {
+                dev_err(ctx->dev, "error %d writing dcs seq: %*ph\n", ret, len,
+                        data);
+                ctx->error = ret;
+        }
+}
+
+static int s6e8aa0_dcs_read(struct s6e8aa0 *ctx, u8 cmd, void *data, size_t len)
+{
+        struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
+        int ret;
+
+        if (ctx->error < 0)
+                return ctx->error;
+
+        ret = mipi_dsi_dcs_read(dsi, dsi->channel, cmd, data, len);
+        if (ret < 0) {
+                dev_err(ctx->dev, "error %d reading dcs seq(%#x)\n", ret, cmd);
+                ctx->error = ret;
+        }
+
+        return ret;
+}
+
+#define s6e8aa0_dcs_write_seq(ctx, seq...) \
+({\
+        const u8 d[] = { seq };\
+        BUILD_BUG_ON_MSG(ARRAY_SIZE(d) > 64, "DCS sequence too big for stack");\
+        s6e8aa0_dcs_write(ctx, d, ARRAY_SIZE(d));\
+})
+
+#define s6e8aa0_dcs_write_seq_static(ctx, seq...) \
+({\
+        static const u8 d[] = { seq };\
+        s6e8aa0_dcs_write(ctx, d, ARRAY_SIZE(d));\
+})
+
+static void s6e8aa0_apply_level_1_key(struct s6e8aa0 *ctx)
+{
+        s6e8aa0_dcs_write_seq_static(ctx, 0xf0, 0x5a, 0x5a);
+}
+
+static void s6e8aa0_panel_cond_set_v142(struct s6e8aa0 *ctx)
+{
+        static const u8 aids[] = {
+                0x04, 0x04, 0x04, 0x04, 0x04, 0x60, 0x80, 0xA0
+        };
+        u8 aid = aids[ctx->id >> 5];
+        u8 cfg = 0x3d;
+        u8 clk_con = 0xc8;
+        u8 int_con = 0x08;
+        u8 bictl_con = 0x48;
+        u8 em_clk1_con = 0xff;
+        u8 em_clk2_con = 0xff;
+        u8 em_int_con = 0xc8;
+
+        if (ctx->flip_vertical) {
+                /* GTCON */
+                cfg &= ~(PANELCTL_GTCON_MASK);
+                cfg |= (PANELCTL_GTCON_110);
+        }
+
+        if (ctx->flip_horizontal) {
+                /* SS */
+                cfg &= ~(PANELCTL_SS_MASK);
+                cfg |= (PANELCTL_SS_1_800);
+        }
+
+        if (ctx->flip_horizontal || ctx->flip_vertical) {
+                /* CLK1,2_CON */
+                clk_con &= ~(PANELCTL_CLK1_CON_MASK |
+                        PANELCTL_CLK2_CON_MASK);
+                clk_con |= (PANELCTL_CLK1_000 | PANELCTL_CLK2_001);
+
+                /* INT1,2_CON */
+                int_con &= ~(PANELCTL_INT1_CON_MASK |
+                        PANELCTL_INT2_CON_MASK);
+                int_con |= (PANELCTL_INT1_000 | PANELCTL_INT2_001);
+
+                /* BICTL,B_CON */
+                bictl_con &= ~(PANELCTL_BICTL_CON_MASK |
+                        PANELCTL_BICTLB_CON_MASK);
+                bictl_con |= (PANELCTL_BICTL_000 |
+                        PANELCTL_BICTLB_001);
+
+                /* EM_CLK1,1B_CON */
+                em_clk1_con &= ~(PANELCTL_EM_CLK1_CON_MASK |
+                        PANELCTL_EM_CLK1B_CON_MASK);
+                em_clk1_con |= (PANELCTL_EM_CLK1_110 |
+                        PANELCTL_EM_CLK1B_110);
+
+                /* EM_CLK2,2B_CON */
+                em_clk2_con &= ~(PANELCTL_EM_CLK2_CON_MASK |
+                        PANELCTL_EM_CLK2B_CON_MASK);
+                em_clk2_con |= (PANELCTL_EM_CLK2_110 |
+                        PANELCTL_EM_CLK2B_110);
+
+                /* EM_INT1,2_CON */
+                em_int_con &= ~(PANELCTL_EM_INT1_CON_MASK |
+                        PANELCTL_EM_INT2_CON_MASK);
+                em_int_con |= (PANELCTL_EM_INT1_000 |
+                        PANELCTL_EM_INT2_001);
+        }
+
+        s6e8aa0_dcs_write_seq(ctx,
+                0xf8, cfg, 0x35, 0x00, 0x00, 0x00, 0x93, 0x00,
+                0x3c, 0x78, 0x08, 0x27, 0x7d, 0x3f, 0x00, 0x00,
+                0x00, 0x20, aid, 0x08, 0x6e, 0x00, 0x00, 0x00,
+                0x02, 0x07, 0x07, 0x23, 0x23, 0xc0, clk_con, int_con,
+                bictl_con, 0xc1, 0x00, 0xc1, em_clk1_con, em_clk2_con,
+                em_int_con);
+}
+
+static void s6e8aa0_panel_cond_set(struct s6e8aa0 *ctx)
+{
+        if (ctx->version < 142)
+                s6e8aa0_dcs_write_seq_static(ctx,
+                        0xf8, 0x19, 0x35, 0x00, 0x00, 0x00, 0x94, 0x00,
+                        0x3c, 0x78, 0x10, 0x27, 0x08, 0x6e, 0x00, 0x00,
+                        0x00, 0x00, 0x04, 0x08, 0x6e, 0x00, 0x00, 0x00,
+                        0x00, 0x07, 0x07, 0x23, 0x6e, 0xc0, 0xc1, 0x01,
+                        0x81, 0xc1, 0x00, 0xc3, 0xf6, 0xf6, 0xc1
+                );
+        else
+                s6e8aa0_panel_cond_set_v142(ctx);
+}
+
+static void s6e8aa0_display_condition_set(struct s6e8aa0 *ctx)
+{
+        s6e8aa0_dcs_write_seq_static(ctx, 0xf2, 0x80, 0x03, 0x0d);
+}
+
+static void s6e8aa0_etc_source_control(struct s6e8aa0 *ctx)
+{
+        s6e8aa0_dcs_write_seq_static(ctx, 0xf6, 0x00, 0x02, 0x00);
+}
+
+static void s6e8aa0_etc_pentile_control(struct s6e8aa0 *ctx)
+{
+        static const u8 pent32[] = {
+                0xb6, 0x0c, 0x02, 0x03, 0x32, 0xc0, 0x44, 0x44, 0xc0, 0x00
+        };
+
+        static const u8 pent142[] = {
+                0xb6, 0x0c, 0x02, 0x03, 0x32, 0xff, 0x44, 0x44, 0xc0, 0x00
+        };
+
+        if (ctx->version < 142)
+                s6e8aa0_dcs_write(ctx, pent32, ARRAY_SIZE(pent32));
+        else
+                s6e8aa0_dcs_write(ctx, pent142, ARRAY_SIZE(pent142));
+}
+
+static void s6e8aa0_etc_power_control(struct s6e8aa0 *ctx)
+{
+        static const u8 pwr142[] = {
+                0xf4, 0xcf, 0x0a, 0x12, 0x10, 0x1e, 0x33, 0x02
+        };
+
+        static const u8 pwr32[] = {
+                0xf4, 0xcf, 0x0a, 0x15, 0x10, 0x19, 0x33, 0x02
+        };
+
+        if (ctx->version < 142)
+                s6e8aa0_dcs_write(ctx, pwr32, ARRAY_SIZE(pwr32));
+        else
+                s6e8aa0_dcs_write(ctx, pwr142, ARRAY_SIZE(pwr142));
+}
+
+static void s6e8aa0_etc_elvss_control(struct s6e8aa0 *ctx)
+{
+        u8 id = ctx->id ? 0 : 0x95;
+
+        s6e8aa0_dcs_write_seq(ctx, 0xb1, 0x04, id);
+}
+
+static void s6e8aa0_elvss_nvm_set_v142(struct s6e8aa0 *ctx)
+{
+        u8 br;
+
+        switch (ctx->brightness) {
+        case 0 ... 6: /* 30cd ~ 100cd */
+                br = 0xdf;
+                break;
+        case 7 ... 11: /* 120cd ~ 150cd */
+                br = 0xdd;
+                break;
+        case 12 ... 15: /* 180cd ~ 210cd */
+        default:
+                br = 0xd9;
+                break;
+        case 16 ... 24: /* 240cd ~ 300cd */
+                br = 0xd0;
+                break;
+        }
+
+        s6e8aa0_dcs_write_seq(ctx, 0xd9, 0x14, 0x40, 0x0c, 0xcb, 0xce, 0x6e,
+                0xc4, 0x0f, 0x40, 0x41, br, 0x00, 0x60, 0x19);
+}
+
+static void s6e8aa0_elvss_nvm_set(struct s6e8aa0 *ctx)
+{
+        if (ctx->version < 142)
+                s6e8aa0_dcs_write_seq_static(ctx,
+                        0xd9, 0x14, 0x40, 0x0c, 0xcb, 0xce, 0x6e, 0xc4, 0x07,
+                        0x40, 0x41, 0xc1, 0x00, 0x60, 0x19);
+        else
+                s6e8aa0_elvss_nvm_set_v142(ctx);
+};
+
+static void s6e8aa0_apply_level_2_key(struct s6e8aa0 *ctx)
+{
+        s6e8aa0_dcs_write_seq_static(ctx, 0xfc, 0x5a, 0x5a);
+}
+
+static const s6e8aa0_gamma_table s6e8aa0_gamma_tables_v142[GAMMA_LEVEL_NUM] = {
+        {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0x62, 0x55, 0x55,
+                0xaf, 0xb1, 0xb1, 0xbd, 0xce, 0xb7, 0x9a, 0xb1,
+                0x90, 0xb2, 0xc4, 0xae, 0x00, 0x60, 0x00, 0x40,
+                0x00, 0x70,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0x74, 0x68, 0x69,
+                0xb8, 0xc1, 0xb7, 0xbd, 0xcd, 0xb8, 0x93, 0xab,
+                0x88, 0xb4, 0xc4, 0xb1, 0x00, 0x6b, 0x00, 0x4d,
+                0x00, 0x7d,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0x95, 0x8a, 0x89,
+                0xb4, 0xc6, 0xb2, 0xc5, 0xd2, 0xbf, 0x90, 0xa8,
+                0x85, 0xb5, 0xc4, 0xb3, 0x00, 0x7b, 0x00, 0x5d,
+                0x00, 0x8f,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0x9f, 0x98, 0x92,
+                0xb3, 0xc4, 0xb0, 0xbc, 0xcc, 0xb4, 0x91, 0xa6,
+                0x87, 0xb5, 0xc5, 0xb4, 0x00, 0x87, 0x00, 0x6a,
+                0x00, 0x9e,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0x99, 0x93, 0x8b,
+                0xb2, 0xc2, 0xb0, 0xbd, 0xce, 0xb4, 0x90, 0xa6,
+                0x87, 0xb3, 0xc3, 0xb2, 0x00, 0x8d, 0x00, 0x70,
+                0x00, 0xa4,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa7, 0xa5, 0x99,
+                0xb2, 0xc2, 0xb0, 0xbb, 0xcd, 0xb1, 0x93, 0xa7,
+                0x8a, 0xb2, 0xc1, 0xb0, 0x00, 0x92, 0x00, 0x75,
+                0x00, 0xaa,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa0, 0xa0, 0x93,
+                0xb6, 0xc4, 0xb4, 0xb5, 0xc8, 0xaa, 0x94, 0xa9,
+                0x8c, 0xb2, 0xc0, 0xb0, 0x00, 0x97, 0x00, 0x7a,
+                0x00, 0xaf,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa3, 0xa7, 0x96,
+                0xb3, 0xc2, 0xb0, 0xba, 0xcb, 0xb0, 0x94, 0xa8,
+                0x8c, 0xb0, 0xbf, 0xaf, 0x00, 0x9f, 0x00, 0x83,
+                0x00, 0xb9,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0x9d, 0xa2, 0x90,
+                0xb6, 0xc5, 0xb3, 0xb8, 0xc9, 0xae, 0x94, 0xa8,
+                0x8d, 0xaf, 0xbd, 0xad, 0x00, 0xa4, 0x00, 0x88,
+                0x00, 0xbf,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa6, 0xac, 0x97,
+                0xb4, 0xc4, 0xb1, 0xbb, 0xcb, 0xb2, 0x93, 0xa7,
+                0x8d, 0xae, 0xbc, 0xad, 0x00, 0xa7, 0x00, 0x8c,
+                0x00, 0xc3,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa2, 0xa9, 0x93,
+                0xb6, 0xc5, 0xb2, 0xba, 0xc9, 0xb0, 0x93, 0xa7,
+                0x8d, 0xae, 0xbb, 0xac, 0x00, 0xab, 0x00, 0x90,
+                0x00, 0xc8,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0x9e, 0xa6, 0x8f,
+                0xb7, 0xc6, 0xb3, 0xb8, 0xc8, 0xb0, 0x93, 0xa6,
+                0x8c, 0xae, 0xbb, 0xad, 0x00, 0xae, 0x00, 0x93,
+                0x00, 0xcc,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xab, 0xb4, 0x9c,
+                0xb3, 0xc3, 0xaf, 0xb7, 0xc7, 0xaf, 0x93, 0xa6,
+                0x8c, 0xaf, 0xbc, 0xad, 0x00, 0xb1, 0x00, 0x97,
+                0x00, 0xcf,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa6, 0xb1, 0x98,
+                0xb1, 0xc2, 0xab, 0xba, 0xc9, 0xb2, 0x93, 0xa6,
+                0x8d, 0xae, 0xba, 0xab, 0x00, 0xb5, 0x00, 0x9b,
+                0x00, 0xd4,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa3, 0xae, 0x94,
+                0xb2, 0xc3, 0xac, 0xbb, 0xca, 0xb4, 0x91, 0xa4,
+                0x8a, 0xae, 0xba, 0xac, 0x00, 0xb8, 0x00, 0x9e,
+                0x00, 0xd8,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xab, 0xb7, 0x9c,
+                0xae, 0xc0, 0xa9, 0xba, 0xc9, 0xb3, 0x92, 0xa5,
+                0x8b, 0xad, 0xb9, 0xab, 0x00, 0xbb, 0x00, 0xa1,
+                0x00, 0xdc,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa7, 0xb4, 0x97,
+                0xb0, 0xc1, 0xaa, 0xb9, 0xc8, 0xb2, 0x92, 0xa5,
+                0x8c, 0xae, 0xb9, 0xab, 0x00, 0xbe, 0x00, 0xa4,
+                0x00, 0xdf,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa3, 0xb0, 0x94,
+                0xb0, 0xc2, 0xab, 0xbb, 0xc9, 0xb3, 0x91, 0xa4,
+                0x8b, 0xad, 0xb8, 0xaa, 0x00, 0xc1, 0x00, 0xa8,
+                0x00, 0xe2,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa3, 0xb0, 0x94,
+                0xae, 0xbf, 0xa8, 0xb9, 0xc8, 0xb3, 0x92, 0xa4,
+                0x8b, 0xad, 0xb7, 0xa9, 0x00, 0xc4, 0x00, 0xab,
+                0x00, 0xe6,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa7, 0xb6, 0x98,
+                0xaf, 0xc0, 0xa8, 0xb8, 0xc7, 0xb2, 0x93, 0xa5,
+                0x8d, 0xad, 0xb7, 0xa9, 0x00, 0xc7, 0x00, 0xae,
+                0x00, 0xe9,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa4, 0xb3, 0x95,
+                0xaf, 0xc1, 0xa9, 0xb9, 0xc8, 0xb3, 0x92, 0xa4,
+                0x8b, 0xad, 0xb7, 0xaa, 0x00, 0xc9, 0x00, 0xb0,
+                0x00, 0xec,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa4, 0xb3, 0x95,
+                0xac, 0xbe, 0xa6, 0xbb, 0xc9, 0xb4, 0x90, 0xa3,
+                0x8a, 0xad, 0xb7, 0xa9, 0x00, 0xcc, 0x00, 0xb4,
+                0x00, 0xf0,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa0, 0xb0, 0x91,
+                0xae, 0xc0, 0xa6, 0xba, 0xc8, 0xb4, 0x91, 0xa4,
+                0x8b, 0xad, 0xb7, 0xa9, 0x00, 0xcf, 0x00, 0xb7,
+                0x00, 0xf3,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa7, 0xb8, 0x98,
+                0xab, 0xbd, 0xa4, 0xbb, 0xc9, 0xb5, 0x91, 0xa3,
+                0x8b, 0xac, 0xb6, 0xa8, 0x00, 0xd1, 0x00, 0xb9,
+                0x00, 0xf6,
+        }, {
+                0xfa, 0x01, 0x71, 0x31, 0x7b, 0xa4, 0xb5, 0x95,
+                0xa9, 0xbc, 0xa1, 0xbb, 0xc9, 0xb5, 0x91, 0xa3,
+                0x8a, 0xad, 0xb6, 0xa8, 0x00, 0xd6, 0x00, 0xbf,
+                0x00, 0xfc,
+        },
+};
+
+static const s6e8aa0_gamma_table s6e8aa0_gamma_tables_v96[GAMMA_LEVEL_NUM] = {
+        {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xff, 0x00, 0xff,
+                0xdf, 0x1f, 0xd7, 0xdc, 0xb7, 0xe1, 0xc0, 0xaf,
+                0xc4, 0xd2, 0xd0, 0xcf, 0x00, 0x4d, 0x00, 0x40,
+                0x00, 0x5f,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xff, 0x00, 0xff,
+                0xd5, 0x35, 0xcf, 0xdc, 0xc1, 0xe1, 0xbf, 0xb3,
+                0xc1, 0xd2, 0xd1, 0xce, 0x00, 0x53, 0x00, 0x46,
+                0x00, 0x67,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xff, 0x00, 0xff,
+                0xd2, 0x64, 0xcf, 0xdb, 0xc6, 0xe1, 0xbd, 0xb3,
+                0xbd, 0xd2, 0xd2, 0xce, 0x00, 0x59, 0x00, 0x4b,
+                0x00, 0x6e,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xff, 0x00, 0xff,
+                0xd0, 0x7c, 0xcf, 0xdb, 0xc9, 0xe0, 0xbc, 0xb4,
+                0xbb, 0xcf, 0xd1, 0xcc, 0x00, 0x5f, 0x00, 0x50,
+                0x00, 0x75,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xff, 0x00, 0xff,
+                0xd0, 0x8e, 0xd1, 0xdb, 0xcc, 0xdf, 0xbb, 0xb6,
+                0xb9, 0xd0, 0xd1, 0xcd, 0x00, 0x63, 0x00, 0x54,
+                0x00, 0x7a,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xff, 0x00, 0xff,
+                0xd1, 0x9e, 0xd5, 0xda, 0xcd, 0xdd, 0xbb, 0xb7,
+                0xb9, 0xce, 0xce, 0xc9, 0x00, 0x68, 0x00, 0x59,
+                0x00, 0x81,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xff, 0x00, 0xff,
+                0xd0, 0xa5, 0xd6, 0xda, 0xcf, 0xdd, 0xbb, 0xb7,
+                0xb8, 0xcc, 0xcd, 0xc7, 0x00, 0x6c, 0x00, 0x5c,
+                0x00, 0x86,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xff, 0x1f, 0xfe,
+                0xd0, 0xae, 0xd7, 0xd9, 0xd0, 0xdb, 0xb9, 0xb6,
+                0xb5, 0xca, 0xcc, 0xc5, 0x00, 0x74, 0x00, 0x63,
+                0x00, 0x90,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xff, 0x1f, 0xf9,
+                0xcf, 0xb0, 0xd6, 0xd9, 0xd1, 0xdb, 0xb9, 0xb6,
+                0xb4, 0xca, 0xcb, 0xc5, 0x00, 0x77, 0x00, 0x66,
+                0x00, 0x94,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xff, 0x1f, 0xf7,
+                0xcf, 0xb3, 0xd7, 0xd8, 0xd1, 0xd9, 0xb7, 0xb6,
+                0xb3, 0xc9, 0xca, 0xc3, 0x00, 0x7b, 0x00, 0x69,
+                0x00, 0x99,
+
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xfd, 0x2f, 0xf7,
+                0xdf, 0xb5, 0xd6, 0xd8, 0xd1, 0xd8, 0xb6, 0xb5,
+                0xb2, 0xca, 0xcb, 0xc4, 0x00, 0x7e, 0x00, 0x6c,
+                0x00, 0x9d,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xfa, 0x2f, 0xf5,
+                0xce, 0xb6, 0xd5, 0xd7, 0xd2, 0xd8, 0xb6, 0xb4,
+                0xb0, 0xc7, 0xc9, 0xc1, 0x00, 0x84, 0x00, 0x71,
+                0x00, 0xa5,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xf7, 0x2f, 0xf2,
+                0xce, 0xb9, 0xd5, 0xd8, 0xd2, 0xd8, 0xb4, 0xb4,
+                0xaf, 0xc7, 0xc9, 0xc1, 0x00, 0x87, 0x00, 0x73,
+                0x00, 0xa8,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xf5, 0x2f, 0xf0,
+                0xdf, 0xba, 0xd5, 0xd7, 0xd2, 0xd7, 0xb4, 0xb4,
+                0xaf, 0xc5, 0xc7, 0xbf, 0x00, 0x8a, 0x00, 0x76,
+                0x00, 0xac,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xf2, 0x2f, 0xed,
+                0xcE, 0xbb, 0xd4, 0xd6, 0xd2, 0xd6, 0xb5, 0xb4,
+                0xaF, 0xc5, 0xc7, 0xbf, 0x00, 0x8c, 0x00, 0x78,
+                0x00, 0xaf,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xef, 0x2f, 0xeb,
+                0xcd, 0xbb, 0xd2, 0xd7, 0xd3, 0xd6, 0xb3, 0xb4,
+                0xae, 0xc5, 0xc6, 0xbe, 0x00, 0x91, 0x00, 0x7d,
+                0x00, 0xb6,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xee, 0x2f, 0xea,
+                0xce, 0xbd, 0xd4, 0xd6, 0xd2, 0xd5, 0xb2, 0xb3,
+                0xad, 0xc3, 0xc4, 0xbb, 0x00, 0x94, 0x00, 0x7f,
+                0x00, 0xba,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xec, 0x2f, 0xe8,
+                0xce, 0xbe, 0xd3, 0xd6, 0xd3, 0xd5, 0xb2, 0xb2,
+                0xac, 0xc3, 0xc5, 0xbc, 0x00, 0x96, 0x00, 0x81,
+                0x00, 0xbd,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xeb, 0x2f, 0xe7,
+                0xce, 0xbf, 0xd3, 0xd6, 0xd2, 0xd5, 0xb1, 0xb2,
+                0xab, 0xc2, 0xc4, 0xbb, 0x00, 0x99, 0x00, 0x83,
+                0x00, 0xc0,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xef, 0x5f, 0xe9,
+                0xca, 0xbf, 0xd3, 0xd5, 0xd2, 0xd4, 0xb2, 0xb2,
+                0xab, 0xc1, 0xc4, 0xba, 0x00, 0x9b, 0x00, 0x85,
+                0x00, 0xc3,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xea, 0x5f, 0xe8,
+                0xee, 0xbf, 0xd2, 0xd5, 0xd2, 0xd4, 0xb1, 0xb2,
+                0xab, 0xc1, 0xc2, 0xb9, 0x00, 0x9D, 0x00, 0x87,
+                0x00, 0xc6,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xe9, 0x5f, 0xe7,
+                0xcd, 0xbf, 0xd2, 0xd6, 0xd2, 0xd4, 0xb1, 0xb2,
+                0xab, 0xbe, 0xc0, 0xb7, 0x00, 0xa1, 0x00, 0x8a,
+                0x00, 0xca,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xe8, 0x61, 0xe6,
+                0xcd, 0xbf, 0xd1, 0xd6, 0xd3, 0xd4, 0xaf, 0xb0,
+                0xa9, 0xbe, 0xc1, 0xb7, 0x00, 0xa3, 0x00, 0x8b,
+                0x00, 0xce,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xe8, 0x62, 0xe5,
+                0xcc, 0xc0, 0xd0, 0xd6, 0xd2, 0xd4, 0xaf, 0xb1,
+                0xa9, 0xbd, 0xc0, 0xb6, 0x00, 0xa5, 0x00, 0x8d,
+                0x00, 0xd0,
+        }, {
+                0xfa, 0x01, 0x1f, 0x1f, 0x1f, 0xe7, 0x7f, 0xe3,
+                0xcc, 0xc1, 0xd0, 0xd5, 0xd3, 0xd3, 0xae, 0xaf,
+                0xa8, 0xbe, 0xc0, 0xb7, 0x00, 0xa8, 0x00, 0x90,
+                0x00, 0xd3,
+        }
+};
+
+static const s6e8aa0_gamma_table s6e8aa0_gamma_tables_v32[GAMMA_LEVEL_NUM] = {
+        {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0x72, 0x5e, 0x6b,
+                0xa1, 0xa7, 0x9a, 0xb4, 0xcb, 0xb8, 0x92, 0xac,
+                0x97, 0xb4, 0xc3, 0xb5, 0x00, 0x4e, 0x00, 0x37,
+                0x00, 0x58,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0x85, 0x71, 0x7d,
+                0xa6, 0xb6, 0xa1, 0xb5, 0xca, 0xba, 0x93, 0xac,
+                0x98, 0xb2, 0xc0, 0xaf, 0x00, 0x59, 0x00, 0x43,
+                0x00, 0x64,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xa4, 0x94, 0x9e,
+                0xa0, 0xbb, 0x9c, 0xc3, 0xd2, 0xc6, 0x93, 0xaa,
+                0x95, 0xb7, 0xc2, 0xb4, 0x00, 0x65, 0x00, 0x50,
+                0x00, 0x74,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xad, 0xa1, 0xa6,
+                0xa0, 0xb9, 0x9b, 0xc3, 0xd1, 0xc8, 0x90, 0xa6,
+                0x90, 0xbb, 0xc3, 0xb7, 0x00, 0x6f, 0x00, 0x5b,
+                0x00, 0x80,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xa6, 0x9d, 0x9f,
+                0x9f, 0xb8, 0x9a, 0xc7, 0xd5, 0xcc, 0x90, 0xa5,
+                0x8f, 0xb8, 0xc1, 0xb6, 0x00, 0x74, 0x00, 0x60,
+                0x00, 0x85,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xb3, 0xae, 0xae,
+                0x9e, 0xb7, 0x9a, 0xc8, 0xd6, 0xce, 0x91, 0xa6,
+                0x90, 0xb6, 0xc0, 0xb3, 0x00, 0x78, 0x00, 0x65,
+                0x00, 0x8a,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xad, 0xa9, 0xa8,
+                0xa3, 0xb9, 0x9e, 0xc4, 0xd3, 0xcb, 0x94, 0xa6,
+                0x90, 0xb6, 0xbf, 0xb3, 0x00, 0x7c, 0x00, 0x69,
+                0x00, 0x8e,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xaf, 0xaf, 0xa9,
+                0xa5, 0xbc, 0xa2, 0xc7, 0xd5, 0xcd, 0x93, 0xa5,
+                0x8f, 0xb4, 0xbd, 0xb1, 0x00, 0x83, 0x00, 0x70,
+                0x00, 0x96,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xa9, 0xab, 0xa3,
+                0xaa, 0xbf, 0xa7, 0xc5, 0xd3, 0xcb, 0x93, 0xa5,
+                0x8f, 0xb2, 0xbb, 0xb0, 0x00, 0x86, 0x00, 0x74,
+                0x00, 0x9b,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xb1, 0xb5, 0xab,
+                0xab, 0xc0, 0xa9, 0xc7, 0xd4, 0xcc, 0x94, 0xa4,
+                0x8f, 0xb1, 0xbb, 0xaf, 0x00, 0x8a, 0x00, 0x77,
+                0x00, 0x9e,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xad, 0xb2, 0xa7,
+                0xae, 0xc2, 0xab, 0xc5, 0xd3, 0xca, 0x93, 0xa4,
+                0x8f, 0xb1, 0xba, 0xae, 0x00, 0x8d, 0x00, 0x7b,
+                0x00, 0xa2,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xa9, 0xaf, 0xa3,
+                0xb0, 0xc3, 0xae, 0xc4, 0xd1, 0xc8, 0x93, 0xa4,
+                0x8f, 0xb1, 0xba, 0xaf, 0x00, 0x8f, 0x00, 0x7d,
+                0x00, 0xa5,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xb4, 0xbd, 0xaf,
+                0xae, 0xc1, 0xab, 0xc2, 0xd0, 0xc6, 0x94, 0xa4,
+                0x8f, 0xb1, 0xba, 0xaf, 0x00, 0x92, 0x00, 0x80,
+                0x00, 0xa8,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xb0, 0xb9, 0xac,
+                0xad, 0xc1, 0xab, 0xc4, 0xd1, 0xc7, 0x95, 0xa4,
+                0x90, 0xb0, 0xb9, 0xad, 0x00, 0x95, 0x00, 0x84,
+                0x00, 0xac,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xad, 0xb6, 0xa7,
+                0xaf, 0xc2, 0xae, 0xc5, 0xd1, 0xc7, 0x93, 0xa3,
+                0x8e, 0xb0, 0xb9, 0xad, 0x00, 0x98, 0x00, 0x86,
+                0x00, 0xaf,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xb4, 0xbf, 0xaf,
+                0xad, 0xc1, 0xab, 0xc3, 0xd0, 0xc6, 0x94, 0xa3,
+                0x8f, 0xaf, 0xb8, 0xac, 0x00, 0x9a, 0x00, 0x89,
+                0x00, 0xb2,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xb0, 0xbc, 0xac,
+                0xaf, 0xc2, 0xad, 0xc2, 0xcf, 0xc4, 0x94, 0xa3,
+                0x90, 0xaf, 0xb8, 0xad, 0x00, 0x9c, 0x00, 0x8b,
+                0x00, 0xb5,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xad, 0xb9, 0xa7,
+                0xb1, 0xc4, 0xaf, 0xc3, 0xcf, 0xc5, 0x94, 0xa3,
+                0x8f, 0xae, 0xb7, 0xac, 0x00, 0x9f, 0x00, 0x8e,
+                0x00, 0xb8,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xad, 0xb9, 0xa7,
+                0xaf, 0xc2, 0xad, 0xc1, 0xce, 0xc3, 0x95, 0xa3,
+                0x90, 0xad, 0xb6, 0xab, 0x00, 0xa2, 0x00, 0x91,
+                0x00, 0xbb,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xb1, 0xbe, 0xac,
+                0xb1, 0xc4, 0xaf, 0xc1, 0xcd, 0xc1, 0x95, 0xa4,
+                0x91, 0xad, 0xb6, 0xab, 0x00, 0xa4, 0x00, 0x93,
+                0x00, 0xbd,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xad, 0xbb, 0xa8,
+                0xb3, 0xc5, 0xb2, 0xc1, 0xcd, 0xc2, 0x95, 0xa3,
+                0x90, 0xad, 0xb6, 0xab, 0x00, 0xa6, 0x00, 0x95,
+                0x00, 0xc0,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xad, 0xbb, 0xa8,
+                0xb0, 0xc3, 0xaf, 0xc2, 0xce, 0xc2, 0x94, 0xa2,
+                0x90, 0xac, 0xb6, 0xab, 0x00, 0xa8, 0x00, 0x98,
+                0x00, 0xc3,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xa9, 0xb8, 0xa5,
+                0xb3, 0xc5, 0xb2, 0xc1, 0xcc, 0xc0, 0x95, 0xa2,
+                0x90, 0xad, 0xb6, 0xab, 0x00, 0xaa, 0x00, 0x9a,
+                0x00, 0xc5,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xb0, 0xc0, 0xac,
+                0xb0, 0xc3, 0xaf, 0xc1, 0xcd, 0xc1, 0x95, 0xa2,
+                0x90, 0xac, 0xb5, 0xa9, 0x00, 0xac, 0x00, 0x9c,
+                0x00, 0xc8,
+        }, {
+                0xfa, 0x01, 0x43, 0x14, 0x45, 0xad, 0xbd, 0xa8,
+                0xaf, 0xc2, 0xaf, 0xc1, 0xcc, 0xc0, 0x95, 0xa2,
+                0x90, 0xac, 0xb5, 0xaa, 0x00, 0xb1, 0x00, 0xa1,
+                0x00, 0xcc,
+        },
+};
+
+static const struct s6e8aa0_variant s6e8aa0_variants[] = {
+        {
+                .version = 32,
+                .gamma_tables = s6e8aa0_gamma_tables_v32,
+        }, {
+                .version = 96,
+                .gamma_tables = s6e8aa0_gamma_tables_v96,
+        }, {
+                .version = 142,
+                .gamma_tables = s6e8aa0_gamma_tables_v142,
+        }, {
+                .version = 210,
+                .gamma_tables = s6e8aa0_gamma_tables_v142,
+        }
+};
+
+static void s6e8aa0_brightness_set(struct s6e8aa0 *ctx)
+{
+        const u8 *gamma;
+
+        if (ctx->error)
+                return;
+
+        gamma = ctx->variant->gamma_tables[ctx->brightness];
+
+        if (ctx->version >= 142)
+                s6e8aa0_elvss_nvm_set(ctx);
+
+        s6e8aa0_dcs_write(ctx, gamma, GAMMA_TABLE_LEN);
+
+        /* update gamma table. */
+        s6e8aa0_dcs_write_seq_static(ctx, 0xf7, 0x03);
+}
+
+static void s6e8aa0_panel_init(struct s6e8aa0 *ctx)
+{
+        s6e8aa0_apply_level_1_key(ctx);
+        s6e8aa0_apply_level_2_key(ctx);
+        msleep(20);
+
+        s6e8aa0_dcs_write_seq_static(ctx, MIPI_DCS_EXIT_SLEEP_MODE);
+        msleep(40);
+
+        s6e8aa0_panel_cond_set(ctx);
+        s6e8aa0_display_condition_set(ctx);
+        s6e8aa0_brightness_set(ctx);
+        s6e8aa0_etc_source_control(ctx);
+        s6e8aa0_etc_pentile_control(ctx);
+        s6e8aa0_elvss_nvm_set(ctx);
+        s6e8aa0_etc_power_control(ctx);
+        s6e8aa0_etc_elvss_control(ctx);
+        msleep(ctx->init_delay);
+}
+
+static void s6e8aa0_set_maximum_return_packet_size(struct s6e8aa0 *ctx,
+                                                   int size)
+{
+        struct mipi_dsi_device *dsi = to_mipi_dsi_device(ctx->dev);
+        const struct mipi_dsi_host_ops *ops = dsi->host->ops;
+        u8 buf[] = {size, 0};
+        struct mipi_dsi_msg msg = {
+                .channel = dsi->channel,
+                .type = MIPI_DSI_SET_MAXIMUM_RETURN_PACKET_SIZE,
+                .tx_len = sizeof(buf),
+                .tx_buf = buf
+        };
+        int ret;
+
+        if (ctx->error < 0)
+                return;
+
+        if (!ops || !ops->transfer)
+                ret = -EIO;
+        else
+                ret = ops->transfer(dsi->host, &msg);
+
+        if (ret < 0) {
+                dev_err(ctx->dev,
+                        "error %d setting maximum return packet size to %d\n",
+                        ret, size);
+                ctx->error = ret;
+        }
+}
+
+static void s6e8aa0_read_mtp_id(struct s6e8aa0 *ctx)
+{
+        u8 id[3];
+        int ret, i;
+
+        ret = s6e8aa0_dcs_read(ctx, 0xd1, id, ARRAY_SIZE(id));
+        if (ret < ARRAY_SIZE(id) || id[0] == 0x00) {
+                dev_err(ctx->dev, "read id failed\n");
+                ctx->error = -EIO;
+                return;
+        }
+
+        dev_info(ctx->dev, "ID: 0x%2x, 0x%2x, 0x%2x\n", id[0], id[1], id[2]);
+
+        for (i = 0; i < ARRAY_SIZE(s6e8aa0_variants); ++i) {
+                if (id[1] == s6e8aa0_variants[i].version)
+                        break;
+        }
+        if (i >= ARRAY_SIZE(s6e8aa0_variants)) {
+                dev_err(ctx->dev, "unsupported display version %d\n", id[1]);
+                ctx->error = -EINVAL;
+        }
+
+        ctx->variant = &s6e8aa0_variants[i];
+        ctx->version = id[1];
+        ctx->id = id[2];
+}
+
+static void s6e8aa0_set_sequence(struct s6e8aa0 *ctx)
+{
+        s6e8aa0_set_maximum_return_packet_size(ctx, 3);
+        s6e8aa0_read_mtp_id(ctx);
+        s6e8aa0_panel_init(ctx);
+        s6e8aa0_dcs_write_seq_static(ctx, MIPI_DCS_SET_DISPLAY_ON);
+}
+
+static int s6e8aa0_power_on(struct s6e8aa0 *ctx)
+{
+        int ret;
+
+        ret = regulator_bulk_enable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+        if (ret < 0)
+                return ret;
+
+        msleep(ctx->power_on_delay);
+
+        gpiod_set_value(ctx->reset_gpio, 0);
+        usleep_range(10000, 11000);
+        gpiod_set_value(ctx->reset_gpio, 1);
+
+        msleep(ctx->reset_delay);
+
+        return 0;
+}
+
+static int s6e8aa0_power_off(struct s6e8aa0 *ctx)
+{
+        return regulator_bulk_disable(ARRAY_SIZE(ctx->supplies), ctx->supplies);
+}
+
+static int s6e8aa0_disable(struct drm_panel *panel)
+{
+        struct s6e8aa0 *ctx = panel_to_s6e8aa0(panel);
+
+        s6e8aa0_dcs_write_seq_static(ctx, MIPI_DCS_ENTER_SLEEP_MODE);
+        s6e8aa0_dcs_write_seq_static(ctx, MIPI_DCS_SET_DISPLAY_OFF);
+        msleep(40);
+
+        s6e8aa0_clear_error(ctx);
+
+        return s6e8aa0_power_off(ctx);
+}
+
+static int s6e8aa0_enable(struct drm_panel *panel)
+{
+        struct s6e8aa0 *ctx = panel_to_s6e8aa0(panel);
+        int ret;
+
+        ret = s6e8aa0_power_on(ctx);
+        if (ret < 0)
+                return ret;
+
+        s6e8aa0_set_sequence(ctx);
+        ret = ctx->error;
+
+        if (ret < 0)
+                s6e8aa0_disable(panel);
+
+        return ret;
+}
+
+static int s6e8aa0_get_modes(struct drm_panel *panel)
+{
+        struct drm_connector *connector = panel->connector;
+        struct s6e8aa0 *ctx = panel_to_s6e8aa0(panel);
+        struct drm_display_mode *mode;
+
+        mode = drm_mode_create(connector->dev);
+        if (!mode) {
+                DRM_ERROR("failed to create a new display mode\n");
+                return 0;
+        }
+
+        drm_display_mode_from_videomode(&ctx->vm, mode);
+        mode->width_mm = ctx->width_mm;
+        mode->height_mm = ctx->height_mm;
+        connector->display_info.width_mm = mode->width_mm;
+        connector->display_info.height_mm = mode->height_mm;
+
+        mode->type = DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED;
+        drm_mode_probed_add(connector, mode);
+
+        return 1;
+}
+
+static const struct drm_panel_funcs s6e8aa0_drm_funcs = {
+        .disable = s6e8aa0_disable,
+        .enable = s6e8aa0_enable,
+        .get_modes = s6e8aa0_get_modes,
+};
+
+static int s6e8aa0_parse_dt(struct s6e8aa0 *ctx)
+{
+        struct device *dev = ctx->dev;
+        struct device_node *np = dev->of_node;
+        int ret;
+
+        ret = of_get_videomode(np, &ctx->vm, 0);
+        if (ret < 0)
+                return ret;
+
+        of_property_read_u32(np, "power-on-delay", &ctx->power_on_delay);
+        of_property_read_u32(np, "reset-delay", &ctx->reset_delay);
+        of_property_read_u32(np, "init-delay", &ctx->init_delay);
+        of_property_read_u32(np, "panel-width-mm", &ctx->width_mm);
+        of_property_read_u32(np, "panel-height-mm", &ctx->height_mm);
+
+        ctx->flip_horizontal = of_property_read_bool(np, "flip-horizontal");
+        ctx->flip_vertical = of_property_read_bool(np, "flip-vertical");
+
+        return 0;
+}
+
+static int s6e8aa0_probe(struct mipi_dsi_device *dsi)
+{
+        struct device *dev = &dsi->dev;
+        struct s6e8aa0 *ctx;
+        int ret;
+
+        ctx = devm_kzalloc(dev, sizeof(struct s6e8aa0), GFP_KERNEL);
+        if (!ctx)
+                return -ENOMEM;
+
+        mipi_dsi_set_drvdata(dsi, ctx);
+
+        ctx->dev = dev;
+
+        dsi->lanes = 4;
+        dsi->format = MIPI_DSI_FMT_RGB888;
+        dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_BURST
+                | MIPI_DSI_MODE_VIDEO_HFP | MIPI_DSI_MODE_VIDEO_HBP
+                | MIPI_DSI_MODE_VIDEO_HSA | MIPI_DSI_MODE_EOT_PACKET
+                | MIPI_DSI_MODE_VSYNC_FLUSH | MIPI_DSI_MODE_VIDEO_AUTO_VERT;
+
+        ret = s6e8aa0_parse_dt(ctx);
+        if (ret < 0)
+                return ret;
+
+        ctx->supplies[0].supply = "vdd3";
+        ctx->supplies[1].supply = "vci";
+        ret = devm_regulator_bulk_get(dev, ARRAY_SIZE(ctx->supplies),
+                                      ctx->supplies);
+        if (ret < 0) {
+                dev_err(dev, "failed to get regulators: %d\n", ret);
+                return ret;
+        }
+
+        ctx->reset_gpio = devm_gpiod_get(dev, "reset");
+        if (IS_ERR(ctx->reset_gpio)) {
+                dev_err(dev, "cannot get reset-gpios %ld\n",
+                        PTR_ERR(ctx->reset_gpio));
+                return PTR_ERR(ctx->reset_gpio);
+        }
+        ret = gpiod_direction_output(ctx->reset_gpio, 1);
+        if (ret < 0) {
+                dev_err(dev, "cannot configure reset-gpios %d\n", ret);
+                return ret;
+        }
+
+        ctx->brightness = GAMMA_LEVEL_NUM - 1;
+
+        drm_panel_init(&ctx->panel);
+        ctx->panel.dev = dev;
+        ctx->panel.funcs = &s6e8aa0_drm_funcs;
+
+        ret = drm_panel_add(&ctx->panel);
+        if (ret < 0)
+                return ret;
+
+        ret = mipi_dsi_attach(dsi);
+        if (ret < 0)
+                drm_panel_remove(&ctx->panel);
+
+        return ret;
+}
+
+static int s6e8aa0_remove(struct mipi_dsi_device *dsi)
+{
+        struct s6e8aa0 *ctx = mipi_dsi_get_drvdata(dsi);
+
+        mipi_dsi_detach(dsi);
+        drm_panel_remove(&ctx->panel);
+
+        return 0;
+}
+
+static struct of_device_id s6e8aa0_of_match[] = {
+        { .compatible = "samsung,s6e8aa0" },
+        { }
+};
+MODULE_DEVICE_TABLE(of, s6e8aa0_of_match);
+
+static struct mipi_dsi_driver s6e8aa0_driver = {
+        .probe = s6e8aa0_probe,
+        .remove = s6e8aa0_remove,
+        .driver = {
+                .name = "panel_s6e8aa0",
+                .owner = THIS_MODULE,
+                .of_match_table = s6e8aa0_of_match,
+        },
+};
+module_mipi_dsi_driver(s6e8aa0_driver);
+
+MODULE_AUTHOR("Donghwa Lee <dh09.lee@samsung.com>");
+MODULE_AUTHOR("Inki Dae <inki.dae@samsung.com>");
+MODULE_AUTHOR("Joongmock Shin <jmock.shin@samsung.com>");
+MODULE_AUTHOR("Eunchul Kim <chulspro.kim@samsung.com>");
+MODULE_AUTHOR("Tomasz Figa <t.figa@samsung.com>");
+MODULE_AUTHOR("Andrzej Hajda <a.hajda@samsung.com>");
+MODULE_DESCRIPTION("MIPI-DSI based s6e8aa0 AMOLED LCD Panel Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index 59d52ca2c67f..309f29e9234a 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -22,9 +22,8 @@
  */
 
 #include <linux/backlight.h>
-#include <linux/gpio.h>
+#include <linux/gpio/consumer.h>
 #include <linux/module.h>
-#include <linux/of_gpio.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/regulator/consumer.h>
@@ -44,9 +43,6 @@ struct panel_desc {
         } size;
 };
 
-/* TODO: convert to gpiod_*() API once it's been merged */
-#define GPIO_ACTIVE_LOW (1 << 0)
-
 struct panel_simple {
         struct drm_panel base;
         bool enabled;
@@ -57,8 +53,7 @@ struct panel_simple {
         struct regulator *supply;
         struct i2c_adapter *ddc;
 
-        unsigned long enable_gpio_flags;
-        int enable_gpio;
+        struct gpio_desc *enable_gpio;
 };
 
 static inline struct panel_simple *to_panel_simple(struct drm_panel *panel)
@@ -110,12 +105,8 @@ static int panel_simple_disable(struct drm_panel *panel)
                 backlight_update_status(p->backlight);
         }
 
-        if (gpio_is_valid(p->enable_gpio)) {
-                if (p->enable_gpio_flags & GPIO_ACTIVE_LOW)
-                        gpio_set_value(p->enable_gpio, 1);
-                else
-                        gpio_set_value(p->enable_gpio, 0);
-        }
+        if (p->enable_gpio)
+                gpiod_set_value_cansleep(p->enable_gpio, 0);
 
         regulator_disable(p->supply);
         p->enabled = false;
@@ -137,12 +128,8 @@ static int panel_simple_enable(struct drm_panel *panel)
                 return err;
         }
 
-        if (gpio_is_valid(p->enable_gpio)) {
-                if (p->enable_gpio_flags & GPIO_ACTIVE_LOW)
-                        gpio_set_value(p->enable_gpio, 0);
-                else
-                        gpio_set_value(p->enable_gpio, 1);
-        }
+        if (p->enable_gpio)
+                gpiod_set_value_cansleep(p->enable_gpio, 1);
 
         if (p->backlight) {
                 p->backlight->props.power = FB_BLANK_UNBLANK;
@@ -185,7 +172,6 @@ static int panel_simple_probe(struct device *dev, const struct panel_desc *desc)
 {
         struct device_node *backlight, *ddc;
         struct panel_simple *panel;
-        enum of_gpio_flags flags;
         int err;
 
         panel = devm_kzalloc(dev, sizeof(*panel), GFP_KERNEL);
@@ -199,29 +185,20 @@ static int panel_simple_probe(struct device *dev, const struct panel_desc *desc)
         if (IS_ERR(panel->supply))
                 return PTR_ERR(panel->supply);
 
-        panel->enable_gpio = of_get_named_gpio_flags(dev->of_node,
-                                                     "enable-gpios", 0,
-                                                     &flags);
-        if (gpio_is_valid(panel->enable_gpio)) {
-                unsigned int value;
-
-                if (flags & OF_GPIO_ACTIVE_LOW)
-                        panel->enable_gpio_flags |= GPIO_ACTIVE_LOW;
-
-                err = gpio_request(panel->enable_gpio, "enable");
-                if (err < 0) {
-                        dev_err(dev, "failed to request GPIO#%u: %d\n",
-                                panel->enable_gpio, err);
+        panel->enable_gpio = devm_gpiod_get(dev, "enable");
+        if (IS_ERR(panel->enable_gpio)) {
+                err = PTR_ERR(panel->enable_gpio);
+                if (err != -ENOENT) {
+                        dev_err(dev, "failed to request GPIO: %d\n", err);
                         return err;
                 }
 
-                value = (panel->enable_gpio_flags & GPIO_ACTIVE_LOW) != 0;
-
-                err = gpio_direction_output(panel->enable_gpio, value);
+                panel->enable_gpio = NULL;
+        } else {
+                err = gpiod_direction_output(panel->enable_gpio, 0);
                 if (err < 0) {
-                        dev_err(dev, "failed to setup GPIO%u: %d\n",
-                                panel->enable_gpio, err);
-                        goto free_gpio;
+                        dev_err(dev, "failed to setup GPIO: %d\n", err);
+                        return err;
                 }
         }
 
@@ -230,10 +207,8 @@ static int panel_simple_probe(struct device *dev, const struct panel_desc *desc)
                 panel->backlight = of_find_backlight_by_node(backlight);
                 of_node_put(backlight);
 
-                if (!panel->backlight) {
-                        err = -EPROBE_DEFER;
-                        goto free_gpio;
-                }
+                if (!panel->backlight)
+                        return -EPROBE_DEFER;
         }
 
         ddc = of_parse_phandle(dev->of_node, "ddc-i2c-bus", 0);
@@ -265,9 +240,6 @@ free_ddc:
 free_backlight:
         if (panel->backlight)
                 put_device(&panel->backlight->dev);
-free_gpio:
-        if (gpio_is_valid(panel->enable_gpio))
-                gpio_free(panel->enable_gpio);
 
         return err;
 }
@@ -287,11 +259,6 @@ static int panel_simple_remove(struct device *dev)
         if (panel->backlight)
                 put_device(&panel->backlight->dev);
 
-        if (gpio_is_valid(panel->enable_gpio))
-                gpio_free(panel->enable_gpio);
-
-        regulator_disable(panel->supply);
-
         return 0;
 }
 
@@ -361,6 +328,28 @@ static const struct panel_desc chunghwa_claa101wb01 = {
         },
 };
 
+static const struct drm_display_mode lg_lp129qe_mode = {
+        .clock = 285250,
+        .hdisplay = 2560,
+        .hsync_start = 2560 + 48,
+        .hsync_end = 2560 + 48 + 32,
+        .htotal = 2560 + 48 + 32 + 80,
+        .vdisplay = 1700,
+        .vsync_start = 1700 + 3,
+        .vsync_end = 1700 + 3 + 10,
+        .vtotal = 1700 + 3 + 10 + 36,
+        .vrefresh = 60,
+};
+
+static const struct panel_desc lg_lp129qe = {
+        .modes = &lg_lp129qe_mode,
+        .num_modes = 1,
+        .size = {
+                .width = 272,
+                .height = 181,
+        },
+};
+
 static const struct drm_display_mode samsung_ltn101nt05_mode = {
         .clock = 54030,
         .hdisplay = 1024,
@@ -394,6 +383,9 @@ static const struct of_device_id platform_of_match[] = {
                 .compatible = "chunghwa,claa101wb01",
                 .data = &chunghwa_claa101wb01
         }, {
+                .compatible = "lg,lp129qe",
+                .data = &lg_lp129qe,
+        }, {
                 .compatible = "samsung,ltn101nt05",
                 .data = &samsung_ltn101nt05,
         }, {
@@ -433,10 +425,65 @@ static struct platform_driver panel_simple_platform_driver = {
 struct panel_desc_dsi {
         struct panel_desc desc;
 
+        unsigned long flags;
         enum mipi_dsi_pixel_format format;
         unsigned int lanes;
 };
 
+static const struct drm_display_mode lg_ld070wx3_sl01_mode = {
+        .clock = 71000,
+        .hdisplay = 800,
+        .hsync_start = 800 + 32,
+        .hsync_end = 800 + 32 + 1,
+        .htotal = 800 + 32 + 1 + 57,
+        .vdisplay = 1280,
+        .vsync_start = 1280 + 28,
+        .vsync_end = 1280 + 28 + 1,
+        .vtotal = 1280 + 28 + 1 + 14,
+        .vrefresh = 60,
+};
+
+static const struct panel_desc_dsi lg_ld070wx3_sl01 = {
+        .desc = {
+                .modes = &lg_ld070wx3_sl01_mode,
+                .num_modes = 1,
+                .size = {
+                        .width = 94,
+                        .height = 151,
+                },
+        },
+        .flags = MIPI_DSI_MODE_VIDEO,
+        .format = MIPI_DSI_FMT_RGB888,
+        .lanes = 4,
+};
+
+static const struct drm_display_mode lg_lh500wx1_sd03_mode = {
+        .clock = 67000,
+        .hdisplay = 720,
+        .hsync_start = 720 + 12,
+        .hsync_end = 720 + 12 + 4,
+        .htotal = 720 + 12 + 4 + 112,
+        .vdisplay = 1280,
+        .vsync_start = 1280 + 8,
+        .vsync_end = 1280 + 8 + 4,
+        .vtotal = 1280 + 8 + 4 + 12,
+        .vrefresh = 60,
+};
+
+static const struct panel_desc_dsi lg_lh500wx1_sd03 = {
+        .desc = {
+                .modes = &lg_lh500wx1_sd03_mode,
+                .num_modes = 1,
+                .size = {
+                        .width = 62,
+                        .height = 110,
+                },
+        },
+        .flags = MIPI_DSI_MODE_VIDEO,
+        .format = MIPI_DSI_FMT_RGB888,
+        .lanes = 4,
+};
+
 static const struct drm_display_mode panasonic_vvx10f004b00_mode = {
         .clock = 157200,
         .hdisplay = 1920,
@@ -459,12 +506,19 @@ static const struct panel_desc_dsi panasonic_vvx10f004b00 = {
                         .height = 136,
                 },
         },
+        .flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_SYNC_PULSE,
         .format = MIPI_DSI_FMT_RGB888,
         .lanes = 4,
 };
 
 static const struct of_device_id dsi_of_match[] = {
         {
+                .compatible = "lg,ld070wx3-sl01",
+                .data = &lg_ld070wx3_sl01
+        }, {
+                .compatible = "lg,lh500wx1-sd03",
+                .data = &lg_lh500wx1_sd03
+        }, {
                 .compatible = "panasonic,vvx10f004b00",
                 .data = &panasonic_vvx10f004b00
         }, {
@@ -489,6 +543,7 @@ static int panel_simple_dsi_probe(struct mipi_dsi_device *dsi)
         if (err < 0)
                 return err;
 
+        dsi->mode_flags = desc->flags;
         dsi->format = desc->format;
         dsi->lanes = desc->lanes;
 
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index 798bde2e5881..41bdd174657e 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -527,7 +527,7 @@ static int qxl_crtc_mode_set(struct drm_crtc *crtc,
         bool recreate_primary = false;
         int ret;
         int surf_id;
-        if (!crtc->fb) {
+        if (!crtc->primary->fb) {
                 DRM_DEBUG_KMS("No FB bound\n");
                 return 0;
         }
@@ -536,7 +536,7 @@ static int qxl_crtc_mode_set(struct drm_crtc *crtc,
                 qfb = to_qxl_framebuffer(old_fb);
                 old_bo = gem_to_qxl_bo(qfb->obj);
         }
-        qfb = to_qxl_framebuffer(crtc->fb);
+        qfb = to_qxl_framebuffer(crtc->primary->fb);
         bo = gem_to_qxl_bo(qfb->obj);
         if (!m)
                 /* and do we care? */
@@ -609,14 +609,14 @@ static void qxl_crtc_disable(struct drm_crtc *crtc)
         struct qxl_crtc *qcrtc = to_qxl_crtc(crtc);
         struct drm_device *dev = crtc->dev;
         struct qxl_device *qdev = dev->dev_private;
-        if (crtc->fb) {
-                struct qxl_framebuffer *qfb = to_qxl_framebuffer(crtc->fb);
+        if (crtc->primary->fb) {
+                struct qxl_framebuffer *qfb = to_qxl_framebuffer(crtc->primary->fb);
                 struct qxl_bo *bo = gem_to_qxl_bo(qfb->obj);
                 int ret;
                 ret = qxl_bo_reserve(bo, false);
                 qxl_bo_unpin(bo);
                 qxl_bo_unreserve(bo);
-                crtc->fb = NULL;
+                crtc->primary->fb = NULL;
         }
 
         qxl_monitors_config_set(qdev, qcrtc->index, 0, 0, 0, 0, 0);
diff --git a/drivers/gpu/drm/qxl/qxl_object.c b/drivers/gpu/drm/qxl/qxl_object.c
index 8691c76c5ef0..b95f144f0b49 100644
--- a/drivers/gpu/drm/qxl/qxl_object.c
+++ b/drivers/gpu/drm/qxl/qxl_object.c
@@ -82,8 +82,6 @@ int qxl_bo_create(struct qxl_device *qdev,
         enum ttm_bo_type type;
         int r;
 
-        if (unlikely(qdev->mman.bdev.dev_mapping == NULL))
-                qdev->mman.bdev.dev_mapping = qdev->ddev->dev_mapping;
         if (kernel)
                 type = ttm_bo_type_kernel;
         else
diff --git a/drivers/gpu/drm/qxl/qxl_release.c b/drivers/gpu/drm/qxl/qxl_release.c
index 821ab7b9409b..14e776f1d14e 100644
--- a/drivers/gpu/drm/qxl/qxl_release.c
+++ b/drivers/gpu/drm/qxl/qxl_release.c
@@ -349,7 +349,7 @@ void qxl_release_fence_buffer_objects(struct qxl_release *release)
                 qxl_fence_add_release_locked(&qbo->fence, release->id);
 
                 ttm_bo_add_to_lru(bo);
-                ww_mutex_unlock(&bo->resv->lock);
+                __ttm_bo_unreserve(bo);
                 entry->reserved = false;
         }
         spin_unlock(&bdev->fence_lock);
diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c
index c7e7e6590c2b..d52c27527b9a 100644
--- a/drivers/gpu/drm/qxl/qxl_ttm.c
+++ b/drivers/gpu/drm/qxl/qxl_ttm.c
@@ -433,6 +433,7 @@ static int qxl_sync_obj_flush(void *sync_obj)
 
 static void qxl_sync_obj_unref(void **sync_obj)
 {
+        *sync_obj = NULL;
 }
 
 static void *qxl_sync_obj_ref(void *sync_obj)
@@ -493,7 +494,9 @@ int qxl_ttm_init(struct qxl_device *qdev)
         /* No others user of address space so set it to 0 */
         r = ttm_bo_device_init(&qdev->mman.bdev,
                                qdev->mman.bo_global_ref.ref.object,
-                               &qxl_bo_driver, DRM_FILE_PAGE_OFFSET, 0);
+                               &qxl_bo_driver,
+                               qdev->ddev->anon_inode->i_mapping,
+                               DRM_FILE_PAGE_OFFSET, 0);
         if (r) {
                 DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
                 return r;
@@ -518,8 +521,6 @@ int qxl_ttm_init(struct qxl_device *qdev)
                  ((unsigned)num_io_pages * PAGE_SIZE) / (1024 * 1024));
         DRM_INFO("qxl: %uM of Surface memory size\n",
                  (unsigned)qdev->surfaceram_size / (1024 * 1024));
-        if (unlikely(qdev->mman.bdev.dev_mapping == NULL))
-                qdev->mman.bdev.dev_mapping = qdev->ddev->dev_mapping;
         r = qxl_ttm_debugfs_init(qdev);
         if (r) {
                 DRM_ERROR("Failed to init debugfs\n");
diff --git a/drivers/gpu/drm/radeon/Makefile b/drivers/gpu/drm/radeon/Makefile
index 306364a1ecda..09433534dc47 100644
--- a/drivers/gpu/drm/radeon/Makefile
+++ b/drivers/gpu/drm/radeon/Makefile
@@ -80,7 +80,7 @@ radeon-y += radeon_device.o radeon_asic.o radeon_kms.o \
         r600_dpm.o rs780_dpm.o rv6xx_dpm.o rv770_dpm.o rv730_dpm.o rv740_dpm.o \
         rv770_smc.o cypress_dpm.o btc_dpm.o sumo_dpm.o sumo_smc.o trinity_dpm.o \
         trinity_smc.o ni_dpm.o si_smc.o si_dpm.o kv_smc.o kv_dpm.o ci_smc.o \
-        ci_dpm.o dce6_afmt.o
+        ci_dpm.o dce6_afmt.o radeon_vm.o
 
 # add async DMA block
 radeon-y += \
@@ -99,6 +99,12 @@ radeon-y += \
         uvd_v3_1.o \
         uvd_v4_2.o
 
+# add VCE block
+radeon-y += \
+        radeon_vce.o \
+        vce_v1_0.o \
+        vce_v2_0.o \
+
 radeon-$(CONFIG_COMPAT) += radeon_ioc32.o
 radeon-$(CONFIG_VGA_SWITCHEROO) += radeon_atpx_handler.o
 radeon-$(CONFIG_ACPI) += radeon_acpi.o
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index daa4dd375ab1..fb187c78978f 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -1106,7 +1106,7 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
         int r;
 
         /* no fb bound */
-        if (!atomic && !crtc->fb) {
+        if (!atomic && !crtc->primary->fb) {
                 DRM_DEBUG_KMS("No FB bound\n");
                 return 0;
         }
@@ -1116,8 +1116,8 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
                 target_fb = fb;
         }
         else {
-                radeon_fb = to_radeon_framebuffer(crtc->fb);
-                target_fb = crtc->fb;
+                radeon_fb = to_radeon_framebuffer(crtc->primary->fb);
+                target_fb = crtc->primary->fb;
         }
 
         /* If atomic, assume fb object is pinned & idle & fenced and
@@ -1316,7 +1316,7 @@ static int dce4_crtc_do_set_base(struct drm_crtc *crtc,
         /* set pageflip to happen anywhere in vblank interval */
         WREG32(EVERGREEN_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 0);
 
-        if (!atomic && fb && fb != crtc->fb) {
+        if (!atomic && fb && fb != crtc->primary->fb) {
                 radeon_fb = to_radeon_framebuffer(fb);
                 rbo = gem_to_radeon_bo(radeon_fb->obj);
                 r = radeon_bo_reserve(rbo, false);
@@ -1350,7 +1350,7 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
         int r;
 
         /* no fb bound */
-        if (!atomic && !crtc->fb) {
+        if (!atomic && !crtc->primary->fb) {
                 DRM_DEBUG_KMS("No FB bound\n");
                 return 0;
         }
@@ -1360,8 +1360,8 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
                 target_fb = fb;
         }
         else {
-                radeon_fb = to_radeon_framebuffer(crtc->fb);
-                target_fb = crtc->fb;
+                radeon_fb = to_radeon_framebuffer(crtc->primary->fb);
+                target_fb = crtc->primary->fb;
         }
 
         obj = radeon_fb->obj;
@@ -1485,7 +1485,7 @@ static int avivo_crtc_do_set_base(struct drm_crtc *crtc,
         /* set pageflip to happen anywhere in vblank interval */
         WREG32(AVIVO_D1MODE_MASTER_UPDATE_MODE + radeon_crtc->crtc_offset, 0);
 
-        if (!atomic && fb && fb != crtc->fb) {
+        if (!atomic && fb && fb != crtc->primary->fb) {
                 radeon_fb = to_radeon_framebuffer(fb);
                 rbo = gem_to_radeon_bo(radeon_fb->obj);
                 r = radeon_bo_reserve(rbo, false);
@@ -1972,12 +1972,12 @@ static void atombios_crtc_disable(struct drm_crtc *crtc)
         int i;
 
         atombios_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
-        if (crtc->fb) {
+        if (crtc->primary->fb) {
                 int r;
                 struct radeon_framebuffer *radeon_fb;
                 struct radeon_bo *rbo;
 
-                radeon_fb = to_radeon_framebuffer(crtc->fb);
+                radeon_fb = to_radeon_framebuffer(crtc->primary->fb);
                 rbo = gem_to_radeon_bo(radeon_fb->obj);
                 r = radeon_bo_reserve(rbo, false);
                 if (unlikely(r))
diff --git a/drivers/gpu/drm/radeon/atombios_dp.c b/drivers/gpu/drm/radeon/atombios_dp.c
index 4ad7643fce5f..8b0ab170cef9 100644
--- a/drivers/gpu/drm/radeon/atombios_dp.c
+++ b/drivers/gpu/drm/radeon/atombios_dp.c
@@ -142,101 +142,69 @@ static int radeon_process_aux_ch(struct radeon_i2c_chan *chan,
         return recv_bytes;
 }
 
-static int radeon_dp_aux_native_write(struct radeon_connector *radeon_connector,
-                                      u16 address, u8 *send, u8 send_bytes, u8 delay)
-{
-        struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
-        int ret;
-        u8 msg[20];
-        int msg_bytes = send_bytes + 4;
-        u8 ack;
-        unsigned retry;
-
-        if (send_bytes > 16)
-                return -1;
+#define HEADER_SIZE 4
 
-        msg[0] = address;
-        msg[1] = address >> 8;
-        msg[2] = DP_AUX_NATIVE_WRITE << 4;
-        msg[3] = (msg_bytes << 4) | (send_bytes - 1);
-        memcpy(&msg[4], send, send_bytes);
-
-        for (retry = 0; retry < 7; retry++) {
-                ret = radeon_process_aux_ch(dig_connector->dp_i2c_bus,
-                                            msg, msg_bytes, NULL, 0, delay, &ack);
-                if (ret == -EBUSY)
-                        continue;
-                else if (ret < 0)
-                        return ret;
-                ack >>= 4;
-                if ((ack & DP_AUX_NATIVE_REPLY_MASK) == DP_AUX_NATIVE_REPLY_ACK)
-                        return send_bytes;
-                else if ((ack & DP_AUX_NATIVE_REPLY_MASK) == DP_AUX_NATIVE_REPLY_DEFER)
-                        usleep_range(400, 500);
-                else
-                        return -EIO;
-        }
-
-        return -EIO;
-}
-
-static int radeon_dp_aux_native_read(struct radeon_connector *radeon_connector,
-                                     u16 address, u8 *recv, int recv_bytes, u8 delay)
+static ssize_t
+radeon_dp_aux_transfer(struct drm_dp_aux *aux, struct drm_dp_aux_msg *msg)
 {
-        struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
-        u8 msg[4];
-        int msg_bytes = 4;
-        u8 ack;
+        struct radeon_i2c_chan *chan =
+                container_of(aux, struct radeon_i2c_chan, aux);
         int ret;
-        unsigned retry;
-
-        msg[0] = address;
-        msg[1] = address >> 8;
-        msg[2] = DP_AUX_NATIVE_READ << 4;
-        msg[3] = (msg_bytes << 4) | (recv_bytes - 1);
-
-        for (retry = 0; retry < 7; retry++) {
-                ret = radeon_process_aux_ch(dig_connector->dp_i2c_bus,
-                                            msg, msg_bytes, recv, recv_bytes, delay, &ack);
-                if (ret == -EBUSY)
-                        continue;
-                else if (ret < 0)
-                        return ret;
-                ack >>= 4;
-                if ((ack & DP_AUX_NATIVE_REPLY_MASK) == DP_AUX_NATIVE_REPLY_ACK)
-                        return ret;
-                else if ((ack & DP_AUX_NATIVE_REPLY_MASK) == DP_AUX_NATIVE_REPLY_DEFER)
-                        usleep_range(400, 500);
-                else if (ret == 0)
-                        return -EPROTO;
-                else
-                        return -EIO;
+        u8 tx_buf[20];
+        size_t tx_size;
+        u8 ack, delay = 0;
+
+        if (WARN_ON(msg->size > 16))
+                return -E2BIG;
+
+        tx_buf[0] = msg->address & 0xff;
+        tx_buf[1] = msg->address >> 8;
+        tx_buf[2] = msg->request << 4;
+        tx_buf[3] = msg->size - 1;
+
+        switch (msg->request & ~DP_AUX_I2C_MOT) {
+        case DP_AUX_NATIVE_WRITE:
+        case DP_AUX_I2C_WRITE:
+                tx_size = HEADER_SIZE + msg->size;
+                tx_buf[3] |= tx_size << 4;
+                memcpy(tx_buf + HEADER_SIZE, msg->buffer, msg->size);
+                ret = radeon_process_aux_ch(chan,
+                                            tx_buf, tx_size, NULL, 0, delay, &ack);
+                if (ret >= 0)
+                        /* Return payload size. */
+                        ret = msg->size;
+                break;
+        case DP_AUX_NATIVE_READ:
+        case DP_AUX_I2C_READ:
+                tx_size = HEADER_SIZE;
+                tx_buf[3] |= tx_size << 4;
+                ret = radeon_process_aux_ch(chan,
+                                            tx_buf, tx_size, msg->buffer, msg->size, delay, &ack);
+                break;
+        default:
+                ret = -EINVAL;
+                break;
         }
 
-        return -EIO;
-}
+        if (ret > 0)
+                msg->reply = ack >> 4;
 
-static void radeon_write_dpcd_reg(struct radeon_connector *radeon_connector,
-                                 u16 reg, u8 val)
-{
-        radeon_dp_aux_native_write(radeon_connector, reg, &val, 1, 0);
+        return ret;
 }
 
-static u8 radeon_read_dpcd_reg(struct radeon_connector *radeon_connector,
-                               u16 reg)
+void radeon_dp_aux_init(struct radeon_connector *radeon_connector)
 {
-        u8 val = 0;
-
-        radeon_dp_aux_native_read(radeon_connector, reg, &val, 1, 0);
+        struct radeon_connector_atom_dig *dig_connector = radeon_connector->con_priv;
 
-        return val;
+        dig_connector->dp_i2c_bus->aux.dev = radeon_connector->base.kdev;
+        dig_connector->dp_i2c_bus->aux.transfer = radeon_dp_aux_transfer;
 }
 
 int radeon_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
                          u8 write_byte, u8 *read_byte)
 {
         struct i2c_algo_dp_aux_data *algo_data = adapter->algo_data;
-        struct radeon_i2c_chan *auxch = (struct radeon_i2c_chan *)adapter;
+        struct radeon_i2c_chan *auxch = i2c_get_adapdata(adapter);
         u16 address = algo_data->address;
         u8 msg[5];
         u8 reply[2];
@@ -246,34 +214,30 @@ int radeon_dp_i2c_aux_ch(struct i2c_adapter *adapter, int mode,
         int ret;
         u8 ack;
 
-        /* Set up the command byte */
-        if (mode & MODE_I2C_READ)
-                msg[2] = DP_AUX_I2C_READ << 4;
-        else
-                msg[2] = DP_AUX_I2C_WRITE << 4;
-
-        if (!(mode & MODE_I2C_STOP))
-                msg[2] |= DP_AUX_I2C_MOT << 4;
-
+        /* Set up the address */
         msg[0] = address;
         msg[1] = address >> 8;
 
-        switch (mode) {
-        case MODE_I2C_WRITE:
+        /* Set up the command byte */
+        if (mode & MODE_I2C_READ) {
+                msg[2] = DP_AUX_I2C_READ << 4;
+                msg_bytes = 4;
+                msg[3] = msg_bytes << 4;
+        } else {
+                msg[2] = DP_AUX_I2C_WRITE << 4;
                 msg_bytes = 5;
                 msg[3] = msg_bytes << 4;
                 msg[4] = write_byte;
-                break;
-        case MODE_I2C_READ:
-                msg_bytes = 4;
-                msg[3] = msg_bytes << 4;
-                break;
-        default:
-                msg_bytes = 4;
-                msg[3] = 3 << 4;
-                break;
         }
 
+        /* special handling for start/stop */
+        if (mode & (MODE_I2C_START | MODE_I2C_STOP))
+                msg[3] = 3 << 4;
+
+        /* Set MOT bit for all but stop */
+        if ((mode & MODE_I2C_STOP) == 0)
+                msg[2] |= DP_AUX_I2C_MOT << 4;
+
         for (retry = 0; retry < 7; retry++) {
                 ret = radeon_process_aux_ch(auxch,
                                             msg, msg_bytes, reply, reply_bytes, 0, &ack);
@@ -472,11 +436,11 @@ static void radeon_dp_probe_oui(struct radeon_connector *radeon_connector)
         if (!(dig_connector->dpcd[DP_DOWN_STREAM_PORT_COUNT] & DP_OUI_SUPPORT))
                 return;
 
-        if (radeon_dp_aux_native_read(radeon_connector, DP_SINK_OUI, buf, 3, 0))
+        if (drm_dp_dpcd_read(&dig_connector->dp_i2c_bus->aux, DP_SINK_OUI, buf, 3))
                 DRM_DEBUG_KMS("Sink OUI: %02hx%02hx%02hx\n",
                               buf[0], buf[1], buf[2]);
 
-        if (radeon_dp_aux_native_read(radeon_connector, DP_BRANCH_OUI, buf, 3, 0))
+        if (drm_dp_dpcd_read(&dig_connector->dp_i2c_bus->aux, DP_BRANCH_OUI, buf, 3))
                 DRM_DEBUG_KMS("Branch OUI: %02hx%02hx%02hx\n",
                               buf[0], buf[1], buf[2]);
 }
@@ -487,8 +451,8 @@ bool radeon_dp_getdpcd(struct radeon_connector *radeon_connector)
         u8 msg[DP_DPCD_SIZE];
         int ret, i;
 
-        ret = radeon_dp_aux_native_read(radeon_connector, DP_DPCD_REV, msg,
-                                        DP_DPCD_SIZE, 0);
+        ret = drm_dp_dpcd_read(&dig_connector->dp_i2c_bus->aux, DP_DPCD_REV, msg,
+                               DP_DPCD_SIZE);
         if (ret > 0) {
                 memcpy(dig_connector->dpcd, msg, DP_DPCD_SIZE);
                 DRM_DEBUG_KMS("DPCD: ");
@@ -510,6 +474,7 @@ int radeon_dp_get_panel_mode(struct drm_encoder *encoder,
         struct drm_device *dev = encoder->dev;
         struct radeon_device *rdev = dev->dev_private;
         struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+        struct radeon_connector_atom_dig *dig_connector;
         int panel_mode = DP_PANEL_MODE_EXTERNAL_DP_MODE;
         u16 dp_bridge = radeon_connector_encoder_get_dp_bridge_encoder_id(connector);
         u8 tmp;
@@ -517,9 +482,15 @@ int radeon_dp_get_panel_mode(struct drm_encoder *encoder,
         if (!ASIC_IS_DCE4(rdev))
                 return panel_mode;
 
+        if (!radeon_connector->con_priv)
+                return panel_mode;
+
+        dig_connector = radeon_connector->con_priv;
+
         if (dp_bridge != ENCODER_OBJECT_ID_NONE) {
                 /* DP bridge chips */
-                tmp = radeon_read_dpcd_reg(radeon_connector, DP_EDP_CONFIGURATION_CAP);
+                drm_dp_dpcd_readb(&dig_connector->dp_i2c_bus->aux,
+                                  DP_EDP_CONFIGURATION_CAP, &tmp);
                 if (tmp & 1)
                         panel_mode = DP_PANEL_MODE_INTERNAL_DP2_MODE;
                 else if ((dp_bridge == ENCODER_OBJECT_ID_NUTMEG) ||
@@ -529,7 +500,8 @@ int radeon_dp_get_panel_mode(struct drm_encoder *encoder,
                         panel_mode = DP_PANEL_MODE_EXTERNAL_DP_MODE;
         } else if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
                 /* eDP */
-                tmp = radeon_read_dpcd_reg(radeon_connector, DP_EDP_CONFIGURATION_CAP);
+                drm_dp_dpcd_readb(&dig_connector->dp_i2c_bus->aux,
+                                  DP_EDP_CONFIGURATION_CAP, &tmp);
                 if (tmp & 1)
                         panel_mode = DP_PANEL_MODE_INTERNAL_DP2_MODE;
         }
@@ -577,37 +549,42 @@ int radeon_dp_mode_valid_helper(struct drm_connector *connector,
         return MODE_OK;
 }
 
-static bool radeon_dp_get_link_status(struct radeon_connector *radeon_connector,
-                                      u8 link_status[DP_LINK_STATUS_SIZE])
-{
-        int ret;
-        ret = radeon_dp_aux_native_read(radeon_connector, DP_LANE0_1_STATUS,
-                                        link_status, DP_LINK_STATUS_SIZE, 100);
-        if (ret <= 0) {
-                return false;
-        }
-
-        DRM_DEBUG_KMS("link status %6ph\n", link_status);
-        return true;
-}
-
 bool radeon_dp_needs_link_train(struct radeon_connector *radeon_connector)
 {
         u8 link_status[DP_LINK_STATUS_SIZE];
         struct radeon_connector_atom_dig *dig = radeon_connector->con_priv;
 
-        if (!radeon_dp_get_link_status(radeon_connector, link_status))
+        if (drm_dp_dpcd_read_link_status(&dig->dp_i2c_bus->aux, link_status) <= 0)
                 return false;
         if (drm_dp_channel_eq_ok(link_status, dig->dp_lane_count))
                 return false;
         return true;
 }
 
+void radeon_dp_set_rx_power_state(struct drm_connector *connector,
+                                  u8 power_state)
+{
+        struct radeon_connector *radeon_connector = to_radeon_connector(connector);
+        struct radeon_connector_atom_dig *dig_connector;
+
+        if (!radeon_connector->con_priv)
+                return;
+
+        dig_connector = radeon_connector->con_priv;
+
+        /* power up/down the sink */
+        if (dig_connector->dpcd[0] >= 0x11) {
+                drm_dp_dpcd_writeb(&dig_connector->dp_i2c_bus->aux,
+                                   DP_SET_POWER, power_state);
+                usleep_range(1000, 2000);
+        }
+}
+
+
 struct radeon_dp_link_train_info {
         struct radeon_device *rdev;
         struct drm_encoder *encoder;
         struct drm_connector *connector;
-        struct radeon_connector *radeon_connector;
         int enc_id;
         int dp_clock;
         int dp_lane_count;
@@ -617,6 +594,7 @@ struct radeon_dp_link_train_info {
         u8 link_status[DP_LINK_STATUS_SIZE];
         u8 tries;
         bool use_dpencoder;
+        struct drm_dp_aux *aux;
 };
 
 static void radeon_dp_update_vs_emph(struct radeon_dp_link_train_info *dp_info)
@@ -627,8 +605,8 @@ static void radeon_dp_update_vs_emph(struct radeon_dp_link_train_info *dp_info)
                                        0, dp_info->train_set[0]); /* sets all lanes at once */
 
         /* set the vs/emph on the sink */
-        radeon_dp_aux_native_write(dp_info->radeon_connector, DP_TRAINING_LANE0_SET,
-                                   dp_info->train_set, dp_info->dp_lane_count, 0);
+        drm_dp_dpcd_write(dp_info->aux, DP_TRAINING_LANE0_SET,
+                          dp_info->train_set, dp_info->dp_lane_count);
 }
 
 static void radeon_dp_set_tp(struct radeon_dp_link_train_info *dp_info, int tp)
@@ -663,7 +641,7 @@ static void radeon_dp_set_tp(struct radeon_dp_link_train_info *dp_info, int tp)
         }
 
         /* enable training pattern on the sink */
-        radeon_write_dpcd_reg(dp_info->radeon_connector, DP_TRAINING_PATTERN_SET, tp);
+        drm_dp_dpcd_writeb(dp_info->aux, DP_TRAINING_PATTERN_SET, tp);
 }
 
 static int radeon_dp_link_train_init(struct radeon_dp_link_train_info *dp_info)
@@ -673,34 +651,30 @@ static int radeon_dp_link_train_init(struct radeon_dp_link_train_info *dp_info)
         u8 tmp;
 
         /* power up the sink */
-        if (dp_info->dpcd[0] >= 0x11) {
-                radeon_write_dpcd_reg(dp_info->radeon_connector,
-                                      DP_SET_POWER, DP_SET_POWER_D0);
-                usleep_range(1000, 2000);
-        }
+        radeon_dp_set_rx_power_state(dp_info->connector, DP_SET_POWER_D0);
 
         /* possibly enable downspread on the sink */
         if (dp_info->dpcd[3] & 0x1)
-                radeon_write_dpcd_reg(dp_info->radeon_connector,
-                                      DP_DOWNSPREAD_CTRL, DP_SPREAD_AMP_0_5);
+                drm_dp_dpcd_writeb(dp_info->aux,
+                                   DP_DOWNSPREAD_CTRL, DP_SPREAD_AMP_0_5);
         else
-                radeon_write_dpcd_reg(dp_info->radeon_connector,
-                                      DP_DOWNSPREAD_CTRL, 0);
+                drm_dp_dpcd_writeb(dp_info->aux,
+                                   DP_DOWNSPREAD_CTRL, 0);
 
         if ((dp_info->connector->connector_type == DRM_MODE_CONNECTOR_eDP) &&
             (dig->panel_mode == DP_PANEL_MODE_INTERNAL_DP2_MODE)) {
-                radeon_write_dpcd_reg(dp_info->radeon_connector, DP_EDP_CONFIGURATION_SET, 1);
+                drm_dp_dpcd_writeb(dp_info->aux, DP_EDP_CONFIGURATION_SET, 1);
         }
 
         /* set the lane count on the sink */
         tmp = dp_info->dp_lane_count;
         if (drm_dp_enhanced_frame_cap(dp_info->dpcd))
                 tmp |= DP_LANE_COUNT_ENHANCED_FRAME_EN;
-        radeon_write_dpcd_reg(dp_info->radeon_connector, DP_LANE_COUNT_SET, tmp);
+        drm_dp_dpcd_writeb(dp_info->aux, DP_LANE_COUNT_SET, tmp);
 
         /* set the link rate on the sink */
         tmp = drm_dp_link_rate_to_bw_code(dp_info->dp_clock);
-        radeon_write_dpcd_reg(dp_info->radeon_connector, DP_LINK_BW_SET, tmp);
+        drm_dp_dpcd_writeb(dp_info->aux, DP_LINK_BW_SET, tmp);
 
         /* start training on the source */
         if (ASIC_IS_DCE4(dp_info->rdev) || !dp_info->use_dpencoder)
@@ -711,9 +685,9 @@ static int radeon_dp_link_train_init(struct radeon_dp_link_train_info *dp_info)
                                           dp_info->dp_clock, dp_info->enc_id, 0);
 
         /* disable the training pattern on the sink */
-        radeon_write_dpcd_reg(dp_info->radeon_connector,
-                              DP_TRAINING_PATTERN_SET,
-                              DP_TRAINING_PATTERN_DISABLE);
+        drm_dp_dpcd_writeb(dp_info->aux,
+                           DP_TRAINING_PATTERN_SET,
+                           DP_TRAINING_PATTERN_DISABLE);
 
         return 0;
 }
@@ -723,9 +697,9 @@ static int radeon_dp_link_train_finish(struct radeon_dp_link_train_info *dp_info
         udelay(400);
 
         /* disable the training pattern on the sink */
-        radeon_write_dpcd_reg(dp_info->radeon_connector,
-                              DP_TRAINING_PATTERN_SET,
-                              DP_TRAINING_PATTERN_DISABLE);
+        drm_dp_dpcd_writeb(dp_info->aux,
+                           DP_TRAINING_PATTERN_SET,
+                           DP_TRAINING_PATTERN_DISABLE);
 
         /* disable the training pattern on the source */
         if (ASIC_IS_DCE4(dp_info->rdev) || !dp_info->use_dpencoder)
@@ -757,7 +731,8 @@ static int radeon_dp_link_train_cr(struct radeon_dp_link_train_info *dp_info)
         while (1) {
                 drm_dp_link_train_clock_recovery_delay(dp_info->dpcd);
 
-                if (!radeon_dp_get_link_status(dp_info->radeon_connector, dp_info->link_status)) {
+                if (drm_dp_dpcd_read_link_status(dp_info->aux,
+                                                 dp_info->link_status) <= 0) {
                         DRM_ERROR("displayport link status failed\n");
                         break;
                 }
@@ -819,7 +794,8 @@ static int radeon_dp_link_train_ce(struct radeon_dp_link_train_info *dp_info)
         while (1) {
                 drm_dp_link_train_channel_eq_delay(dp_info->dpcd);
 
-                if (!radeon_dp_get_link_status(dp_info->radeon_connector, dp_info->link_status)) {
+                if (drm_dp_dpcd_read_link_status(dp_info->aux,
+                                                 dp_info->link_status) <= 0) {
                         DRM_ERROR("displayport link status failed\n");
                         break;
                 }
@@ -902,7 +878,7 @@ void radeon_dp_link_train(struct drm_encoder *encoder,
         else
                 dp_info.enc_id |= ATOM_DP_CONFIG_LINK_A;
 
-        tmp = radeon_read_dpcd_reg(radeon_connector, DP_MAX_LANE_COUNT);
+        drm_dp_dpcd_readb(&dig_connector->dp_i2c_bus->aux, DP_MAX_LANE_COUNT, &tmp);
         if (ASIC_IS_DCE5(rdev) && (tmp & DP_TPS3_SUPPORTED))
                 dp_info.tp3_supported = true;
         else
@@ -912,9 +888,9 @@ void radeon_dp_link_train(struct drm_encoder *encoder,
         dp_info.rdev = rdev;
         dp_info.encoder = encoder;
         dp_info.connector = connector;
-        dp_info.radeon_connector = radeon_connector;
         dp_info.dp_lane_count = dig_connector->dp_lane_count;
         dp_info.dp_clock = dig_connector->dp_clock;
+        dp_info.aux = &dig_connector->dp_i2c_bus->aux;
 
         if (radeon_dp_link_train_init(&dp_info))
                 goto done;
diff --git a/drivers/gpu/drm/radeon/atombios_encoders.c b/drivers/gpu/drm/radeon/atombios_encoders.c
index 607dc14d195e..e6eb5097597f 100644
--- a/drivers/gpu/drm/radeon/atombios_encoders.c
+++ b/drivers/gpu/drm/radeon/atombios_encoders.c
@@ -1633,10 +1633,16 @@ radeon_atom_encoder_dpms_dig(struct drm_encoder *encoder, int mode)
         struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
         struct radeon_connector *radeon_connector = NULL;
         struct radeon_connector_atom_dig *radeon_dig_connector = NULL;
+        bool travis_quirk = false;
 
         if (connector) {
                 radeon_connector = to_radeon_connector(connector);
                 radeon_dig_connector = radeon_connector->con_priv;
+                if ((radeon_connector_encoder_get_dp_bridge_encoder_id(connector) ==
+                     ENCODER_OBJECT_ID_TRAVIS) &&
+                    (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT)) &&
+                    !ASIC_IS_DCE5(rdev))
+                        travis_quirk = true;
         }
 
         switch (mode) {
@@ -1657,17 +1663,13 @@ radeon_atom_encoder_dpms_dig(struct drm_encoder *encoder, int mode)
                                         atombios_external_encoder_setup(encoder, ext_encoder,
                                                                         EXTERNAL_ENCODER_ACTION_V3_ENCODER_SETUP);
                         }
-                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE, 0, 0);
                 } else if (ASIC_IS_DCE4(rdev)) {
                         /* setup and enable the encoder */
                         atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_SETUP, 0);
-                        /* enable the transmitter */
-                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE, 0, 0);
                 } else {
                         /* setup and enable the encoder and transmitter */
                         atombios_dig_encoder_setup(encoder, ATOM_ENABLE, 0);
                         atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_SETUP, 0, 0);
-                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE, 0, 0);
                 }
                 if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(encoder)) && connector) {
                         if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
@@ -1675,68 +1677,56 @@ radeon_atom_encoder_dpms_dig(struct drm_encoder *encoder, int mode)
                                                              ATOM_TRANSMITTER_ACTION_POWER_ON);
                                 radeon_dig_connector->edp_on = true;
                         }
+                }
+                /* enable the transmitter */
+                atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_ENABLE, 0, 0);
+                if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(encoder)) && connector) {
+                        /* DP_SET_POWER_D0 is set in radeon_dp_link_train */
                         radeon_dp_link_train(encoder, connector);
                         if (ASIC_IS_DCE4(rdev))
                                 atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_VIDEO_ON, 0);
                 }
                 if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
-                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_LCD_BLON, 0, 0);
+                        atombios_dig_transmitter_setup(encoder,
+                                                       ATOM_TRANSMITTER_ACTION_LCD_BLON, 0, 0);
+                if (ext_encoder)
+                        atombios_external_encoder_setup(encoder, ext_encoder, ATOM_ENABLE);
                 break;
         case DRM_MODE_DPMS_STANDBY:
         case DRM_MODE_DPMS_SUSPEND:
         case DRM_MODE_DPMS_OFF:
                 if (ASIC_IS_DCE4(rdev)) {
+                        if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(encoder)) && connector)
+                                atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_VIDEO_OFF, 0);
+                }
+                if (ext_encoder)
+                        atombios_external_encoder_setup(encoder, ext_encoder, ATOM_DISABLE);
+                if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
+                        atombios_dig_transmitter_setup(encoder,
+                                                       ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0);
+
+                if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(encoder)) &&
+                    connector && !travis_quirk)
+                        radeon_dp_set_rx_power_state(connector, DP_SET_POWER_D3);
+                if (ASIC_IS_DCE4(rdev)) {
                         /* disable the transmitter */
-                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0);
+                        atombios_dig_transmitter_setup(encoder,
+                                                       ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0);
                 } else {
                         /* disable the encoder and transmitter */
-                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0);
+                        atombios_dig_transmitter_setup(encoder,
+                                                       ATOM_TRANSMITTER_ACTION_DISABLE, 0, 0);
                         atombios_dig_encoder_setup(encoder, ATOM_DISABLE, 0);
                 }
                 if (ENCODER_MODE_IS_DP(atombios_get_encoder_mode(encoder)) && connector) {
-                        if (ASIC_IS_DCE4(rdev))
-                                atombios_dig_encoder_setup(encoder, ATOM_ENCODER_CMD_DP_VIDEO_OFF, 0);
+                        if (travis_quirk)
+                                radeon_dp_set_rx_power_state(connector, DP_SET_POWER_D3);
                         if (connector->connector_type == DRM_MODE_CONNECTOR_eDP) {
                                 atombios_set_edp_panel_power(connector,
                                                              ATOM_TRANSMITTER_ACTION_POWER_OFF);
                                 radeon_dig_connector->edp_on = false;
                         }
                 }
-                if (radeon_encoder->devices & (ATOM_DEVICE_LCD_SUPPORT))
-                        atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0);
-                break;
-        }
-}
-
-static void
-radeon_atom_encoder_dpms_ext(struct drm_encoder *encoder,
-                             struct drm_encoder *ext_encoder,
-                             int mode)
-{
-        struct drm_device *dev = encoder->dev;
-        struct radeon_device *rdev = dev->dev_private;
-
-        switch (mode) {
-        case DRM_MODE_DPMS_ON:
-        default:
-                if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE61(rdev)) {
-                        atombios_external_encoder_setup(encoder, ext_encoder,
-                                                        EXTERNAL_ENCODER_ACTION_V3_ENABLE_OUTPUT);
-                        atombios_external_encoder_setup(encoder, ext_encoder,
-                                                        EXTERNAL_ENCODER_ACTION_V3_ENCODER_BLANKING_OFF);
-                } else
-                        atombios_external_encoder_setup(encoder, ext_encoder, ATOM_ENABLE);
-                break;
-        case DRM_MODE_DPMS_STANDBY:
-        case DRM_MODE_DPMS_SUSPEND:
-        case DRM_MODE_DPMS_OFF:
-                if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE61(rdev)) {
-                        atombios_external_encoder_setup(encoder, ext_encoder,
-                                                        EXTERNAL_ENCODER_ACTION_V3_ENCODER_BLANKING);
-                        atombios_external_encoder_setup(encoder, ext_encoder,
-                                                        EXTERNAL_ENCODER_ACTION_V3_DISABLE_OUTPUT);
-                } else
-                        atombios_external_encoder_setup(encoder, ext_encoder, ATOM_DISABLE);
                 break;
         }
 }
@@ -1747,7 +1737,6 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
         struct drm_device *dev = encoder->dev;
         struct radeon_device *rdev = dev->dev_private;
         struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
-        struct drm_encoder *ext_encoder = radeon_get_external_encoder(encoder);
 
         DRM_DEBUG_KMS("encoder dpms %d to mode %d, devices %08x, active_devices %08x\n",
                   radeon_encoder->encoder_id, mode, radeon_encoder->devices,
@@ -1807,9 +1796,6 @@ radeon_atom_encoder_dpms(struct drm_encoder *encoder, int mode)
                 return;
         }
 
-        if (ext_encoder)
-                radeon_atom_encoder_dpms_ext(encoder, ext_encoder, mode);
-
         radeon_atombios_encoder_dpms_scratch_regs(encoder, (mode == DRM_MODE_DPMS_ON) ? true : false);
 
 }
diff --git a/drivers/gpu/drm/radeon/btc_dpm.c b/drivers/gpu/drm/radeon/btc_dpm.c
index ea103ccdf4bd..f81d7ca134db 100644
--- a/drivers/gpu/drm/radeon/btc_dpm.c
+++ b/drivers/gpu/drm/radeon/btc_dpm.c
@@ -2601,6 +2601,10 @@ int btc_dpm_init(struct radeon_device *rdev)
         pi->min_vddc_in_table = 0;
         pi->max_vddc_in_table = 0;
 
+        ret = r600_get_platform_caps(rdev);
+        if (ret)
+                return ret;
+
         ret = rv7xx_parse_power_table(rdev);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c
index 8d49104ca6c2..cad89a977527 100644
--- a/drivers/gpu/drm/radeon/ci_dpm.c
+++ b/drivers/gpu/drm/radeon/ci_dpm.c
@@ -172,6 +172,8 @@ extern void si_trim_voltage_table_to_fit_state_table(struct radeon_device *rdev,
 extern void cik_enter_rlc_safe_mode(struct radeon_device *rdev);
 extern void cik_exit_rlc_safe_mode(struct radeon_device *rdev);
 extern int ci_mc_load_microcode(struct radeon_device *rdev);
+extern void cik_update_cg(struct radeon_device *rdev,
+                          u32 block, bool enable);
 
 static int ci_get_std_voltage_value_sidd(struct radeon_device *rdev,
                                          struct atom_voltage_table_entry *voltage_table,
@@ -746,6 +748,14 @@ static void ci_apply_state_adjust_rules(struct radeon_device *rdev,
         u32 max_sclk_vddc, max_mclk_vddci, max_mclk_vddc;
         int i;
 
+        if (rps->vce_active) {
+                rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk;
+                rps->ecclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].ecclk;
+        } else {
+                rps->evclk = 0;
+                rps->ecclk = 0;
+        }
+
         if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
             ci_dpm_vblank_too_short(rdev))
                 disable_mclk_switching = true;
@@ -804,6 +814,13 @@ static void ci_apply_state_adjust_rules(struct radeon_device *rdev,
                 sclk = ps->performance_levels[0].sclk;
         }
 
+        if (rps->vce_active) {
+                if (sclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk)
+                        sclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk;
+                if (mclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].mclk)
+                        mclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].mclk;
+        }
+
         ps->performance_levels[0].sclk = sclk;
         ps->performance_levels[0].mclk = mclk;
 
@@ -3468,7 +3485,6 @@ static int ci_enable_uvd_dpm(struct radeon_device *rdev, bool enable)
                 0 : -EINVAL;
 }
 
-#if 0
 static int ci_enable_vce_dpm(struct radeon_device *rdev, bool enable)
 {
         struct ci_power_info *pi = ci_get_pi(rdev);
@@ -3501,6 +3517,7 @@ static int ci_enable_vce_dpm(struct radeon_device *rdev, bool enable)
                 0 : -EINVAL;
 }
 
+#if 0
 static int ci_enable_samu_dpm(struct radeon_device *rdev, bool enable)
 {
         struct ci_power_info *pi = ci_get_pi(rdev);
@@ -3587,7 +3604,6 @@ static int ci_update_uvd_dpm(struct radeon_device *rdev, bool gate)
         return ci_enable_uvd_dpm(rdev, !gate);
 }
 
-#if 0
 static u8 ci_get_vce_boot_level(struct radeon_device *rdev)
 {
         u8 i;
@@ -3608,15 +3624,15 @@ static int ci_update_vce_dpm(struct radeon_device *rdev,
                              struct radeon_ps *radeon_current_state)
 {
         struct ci_power_info *pi = ci_get_pi(rdev);
-        bool new_vce_clock_non_zero = (radeon_new_state->evclk != 0);
-        bool old_vce_clock_non_zero = (radeon_current_state->evclk != 0);
         int ret = 0;
         u32 tmp;
 
-        if (new_vce_clock_non_zero != old_vce_clock_non_zero) {
-                if (new_vce_clock_non_zero) {
-                        pi->smc_state_table.VceBootLevel = ci_get_vce_boot_level(rdev);
+        if (radeon_current_state->evclk != radeon_new_state->evclk) {
+                if (radeon_new_state->evclk) {
+                        /* turn the clocks on when encoding */
+                        cik_update_cg(rdev, RADEON_CG_BLOCK_VCE, false);
 
+                        pi->smc_state_table.VceBootLevel = ci_get_vce_boot_level(rdev);
                         tmp = RREG32_SMC(DPM_TABLE_475);
                         tmp &= ~VceBootLevel_MASK;
                         tmp |= VceBootLevel(pi->smc_state_table.VceBootLevel);
@@ -3624,12 +3640,16 @@ static int ci_update_vce_dpm(struct radeon_device *rdev,
 
                         ret = ci_enable_vce_dpm(rdev, true);
                 } else {
+                        /* turn the clocks off when not encoding */
+                        cik_update_cg(rdev, RADEON_CG_BLOCK_VCE, true);
+
                         ret = ci_enable_vce_dpm(rdev, false);
                 }
         }
         return ret;
 }
 
+#if 0
 static int ci_update_samu_dpm(struct radeon_device *rdev, bool gate)
 {
         return ci_enable_samu_dpm(rdev, gate);
@@ -4752,13 +4772,13 @@ int ci_dpm_set_power_state(struct radeon_device *rdev)
                 DRM_ERROR("ci_generate_dpm_level_enable_mask failed\n");
                 return ret;
         }
-#if 0
+
         ret = ci_update_vce_dpm(rdev, new_ps, old_ps);
         if (ret) {
                 DRM_ERROR("ci_update_vce_dpm failed\n");
                 return ret;
         }
-#endif
+
         ret = ci_update_sclk_t(rdev);
         if (ret) {
                 DRM_ERROR("ci_update_sclk_t failed\n");
@@ -4959,9 +4979,6 @@ static int ci_parse_power_table(struct radeon_device *rdev)
         if (!rdev->pm.dpm.ps)
                 return -ENOMEM;
         power_state_offset = (u8 *)state_array->states;
-        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
-        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
-        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
         for (i = 0; i < state_array->ucNumEntries; i++) {
                 u8 *idx;
                 power_state = (union pplib_power_state *)power_state_offset;
@@ -4998,6 +5015,21 @@ static int ci_parse_power_table(struct radeon_device *rdev)
                 power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
         }
         rdev->pm.dpm.num_ps = state_array->ucNumEntries;
+
+        /* fill in the vce power states */
+        for (i = 0; i < RADEON_MAX_VCE_LEVELS; i++) {
+                u32 sclk, mclk;
+                clock_array_index = rdev->pm.dpm.vce_states[i].clk_idx;
+                clock_info = (union pplib_clock_info *)
+                        &clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
+                sclk = le16_to_cpu(clock_info->ci.usEngineClockLow);
+                sclk |= clock_info->ci.ucEngineClockHigh << 16;
+                mclk = le16_to_cpu(clock_info->ci.usMemoryClockLow);
+                mclk |= clock_info->ci.ucMemoryClockHigh << 16;
+                rdev->pm.dpm.vce_states[i].sclk = sclk;
+                rdev->pm.dpm.vce_states[i].mclk = mclk;
+        }
+
         return 0;
 }
 
@@ -5077,17 +5109,25 @@ int ci_dpm_init(struct radeon_device *rdev)
                 ci_dpm_fini(rdev);
                 return ret;
         }
-        ret = ci_parse_power_table(rdev);
+
+        ret = r600_get_platform_caps(rdev);
         if (ret) {
                 ci_dpm_fini(rdev);
                 return ret;
         }
+
         ret = r600_parse_extended_power_table(rdev);
         if (ret) {
                 ci_dpm_fini(rdev);
                 return ret;
         }
 
+        ret = ci_parse_power_table(rdev);
+        if (ret) {
+                ci_dpm_fini(rdev);
+                return ret;
+        }
+
         pi->dll_default_on = false;
         pi->sram_end = SMC_RAM_END;
 
@@ -5120,6 +5160,7 @@ int ci_dpm_init(struct radeon_device *rdev)
         pi->caps_sclk_throttle_low_notification = false;
 
         pi->caps_uvd_dpm = true;
+        pi->caps_vce_dpm = true;
 
         ci_get_leakage_voltages(rdev);
         ci_patch_dependency_tables_with_leakage(rdev);
diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c
index bbb17841a9e5..745143c2358f 100644
--- a/drivers/gpu/drm/radeon/cik.c
+++ b/drivers/gpu/drm/radeon/cik.c
@@ -75,6 +75,7 @@ extern void si_init_uvd_internal_cg(struct radeon_device *rdev);
 extern int cik_sdma_resume(struct radeon_device *rdev);
 extern void cik_sdma_enable(struct radeon_device *rdev, bool enable);
 extern void cik_sdma_fini(struct radeon_device *rdev);
+extern void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable);
 static void cik_rlc_stop(struct radeon_device *rdev);
 static void cik_pcie_gen3_enable(struct radeon_device *rdev);
 static void cik_program_aspm(struct radeon_device *rdev);
@@ -1095,7 +1096,7 @@ static const u32 spectre_golden_registers[] =
         0x8a14, 0xf000003f, 0x00000007,
         0x8b24, 0xffffffff, 0x00ffffff,
         0x28350, 0x3f3f3fff, 0x00000082,
-        0x28355, 0x0000003f, 0x00000000,
+        0x28354, 0x0000003f, 0x00000000,
         0x3e78, 0x00000001, 0x00000002,
         0x913c, 0xffff03df, 0x00000004,
         0xc768, 0x00000008, 0x00000008,
@@ -2028,6 +2029,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                 break;
                         case 5:
                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
                                 break;
                         case 6:
@@ -2048,6 +2050,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                 break;
                         case 9:
                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
                                 break;
                         case 10:
@@ -2070,6 +2073,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                 break;
                         case 13:
                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
                                 break;
                         case 14:
@@ -2092,6 +2096,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                 break;
                         case 27:
                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 PIPE_CONFIG(ADDR_SURF_P16_32x32_16x16) |
                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
                                 break;
                         case 28:
@@ -2246,6 +2251,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                 break;
                         case 5:
                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
                                 break;
                         case 6:
@@ -2266,6 +2272,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                 break;
                         case 9:
                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
                                 break;
                         case 10:
@@ -2288,6 +2295,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                 break;
                         case 13:
                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
                                 break;
                         case 14:
@@ -2310,6 +2318,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                 break;
                         case 27:
                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 PIPE_CONFIG(ADDR_SURF_P8_32x32_16x16) |
                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
                                 break;
                         case 28:
@@ -2466,6 +2475,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                         break;
                                 case 5:
                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
                                         break;
                                 case 6:
@@ -2486,6 +2496,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                         break;
                                 case 9:
                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
                                         break;
                                 case 10:
@@ -2508,6 +2519,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                         break;
                                 case 13:
                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
                                         break;
                                 case 14:
@@ -2530,6 +2542,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                         break;
                                 case 27:
                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_16x16) |
                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
                                         break;
                                 case 28:
@@ -2592,6 +2605,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                         break;
                                 case 5:
                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
                                         break;
                                 case 6:
@@ -2612,6 +2626,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                         break;
                                 case 9:
                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
                                         break;
                                 case 10:
@@ -2634,6 +2649,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                         break;
                                 case 13:
                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
                                         break;
                                 case 14:
@@ -2656,6 +2672,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                         break;
                                 case 27:
                                         gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                         PIPE_CONFIG(ADDR_SURF_P4_8x16) |
                                                          MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
                                         break;
                                 case 28:
@@ -2812,6 +2829,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                 break;
                         case 5:
                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_DEPTH_MICRO_TILING));
                                 break;
                         case 6:
@@ -2827,11 +2845,13 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                                  TILE_SPLIT(split_equal_to_row_size));
                                 break;
                         case 8:
-                                gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED);
+                                gb_tile_moden = ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
+                                                PIPE_CONFIG(ADDR_SURF_P2);
                                 break;
                         case 9:
                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING));
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_DISPLAY_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2));
                                 break;
                         case 10:
                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
@@ -2853,6 +2873,7 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                 break;
                         case 13:
                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2) |
                                                  MICRO_TILE_MODE_NEW(ADDR_SURF_THIN_MICRO_TILING));
                                 break;
                         case 14:
@@ -2875,7 +2896,8 @@ static void cik_tiling_mode_table_init(struct radeon_device *rdev)
                                 break;
                         case 27:
                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
-                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING));
+                                                 MICRO_TILE_MODE_NEW(ADDR_SURF_ROTATED_MICRO_TILING) |
+                                                 PIPE_CONFIG(ADDR_SURF_P2));
                                 break;
                         case 28:
                                 gb_tile_moden = (ARRAY_MODE(ARRAY_PRT_2D_TILED_THIN1) |
@@ -4030,8 +4052,6 @@ static int cik_cp_gfx_resume(struct radeon_device *rdev)
         WREG32(CP_RB0_BASE, rb_addr);
         WREG32(CP_RB0_BASE_HI, upper_32_bits(rb_addr));
 
-        ring->rptr = RREG32(CP_RB0_RPTR);
-
         /* start the ring */
         cik_cp_gfx_start(rdev);
         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
@@ -4589,8 +4609,7 @@ static int cik_cp_compute_resume(struct radeon_device *rdev)
                 rdev->ring[idx].wptr = 0;
                 mqd->queue_state.cp_hqd_pq_wptr = rdev->ring[idx].wptr;
                 WREG32(CP_HQD_PQ_WPTR, mqd->queue_state.cp_hqd_pq_wptr);
-                rdev->ring[idx].rptr = RREG32(CP_HQD_PQ_RPTR);
-                mqd->queue_state.cp_hqd_pq_rptr = rdev->ring[idx].rptr;
+                mqd->queue_state.cp_hqd_pq_rptr = RREG32(CP_HQD_PQ_RPTR);
 
                 /* set the vmid for the queue */
                 mqd->queue_state.cp_hqd_vmid = 0;
@@ -5120,11 +5139,9 @@ bool cik_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
         if (!(reset_mask & (RADEON_RESET_GFX |
                             RADEON_RESET_COMPUTE |
                             RADEON_RESET_CP))) {
-                radeon_ring_lockup_update(ring);
+                radeon_ring_lockup_update(rdev, ring);
                 return false;
         }
-        /* force CP activities */
-        radeon_ring_force_activity(rdev, ring);
         return radeon_ring_test_lockup(rdev, ring);
 }
 
@@ -6144,6 +6161,10 @@ void cik_update_cg(struct radeon_device *rdev,
                 cik_enable_hdp_mgcg(rdev, enable);
                 cik_enable_hdp_ls(rdev, enable);
         }
+
+        if (block & RADEON_CG_BLOCK_VCE) {
+                vce_v2_0_enable_mgcg(rdev, enable);
+        }
 }
 
 static void cik_init_cg(struct radeon_device *rdev)
@@ -6521,8 +6542,8 @@ void cik_get_csb_buffer(struct radeon_device *rdev, volatile u32 *buffer)
                 buffer[count++] = cpu_to_le32(0x00000000);
                 break;
         case CHIP_HAWAII:
-                buffer[count++] = 0x3a00161a;
-                buffer[count++] = 0x0000002e;
+                buffer[count++] = cpu_to_le32(0x3a00161a);
+                buffer[count++] = cpu_to_le32(0x0000002e);
                 break;
         default:
                 buffer[count++] = cpu_to_le32(0x00000000);
@@ -7493,6 +7514,20 @@ restart_ih:
                         /* reset addr and status */
                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
                         break;
+                case 167: /* VCE */
+                        DRM_DEBUG("IH: VCE int: 0x%08x\n", src_data);
+                        switch (src_data) {
+                        case 0:
+                                radeon_fence_process(rdev, TN_RING_TYPE_VCE1_INDEX);
+                                break;
+                        case 1:
+                                radeon_fence_process(rdev, TN_RING_TYPE_VCE2_INDEX);
+                                break;
+                        default:
+                                DRM_ERROR("Unhandled interrupt: %d %d\n", src_id, src_data);
+                                break;
+                        }
+                        break;
                 case 176: /* GFX RB CP_INT */
                 case 177: /* GFX IB CP_INT */
                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
@@ -7792,6 +7827,22 @@ static int cik_startup(struct radeon_device *rdev)
         if (r)
                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
 
+        r = radeon_vce_resume(rdev);
+        if (!r) {
+                r = vce_v2_0_resume(rdev);
+                if (!r)
+                        r = radeon_fence_driver_start_ring(rdev,
+                                                           TN_RING_TYPE_VCE1_INDEX);
+                if (!r)
+                        r = radeon_fence_driver_start_ring(rdev,
+                                                           TN_RING_TYPE_VCE2_INDEX);
+        }
+        if (r) {
+                dev_err(rdev->dev, "VCE init error (%d).\n", r);
+                rdev->ring[TN_RING_TYPE_VCE1_INDEX].ring_size = 0;
+                rdev->ring[TN_RING_TYPE_VCE2_INDEX].ring_size = 0;
+        }
+
         /* Enable IRQ */
         if (!rdev->irq.installed) {
                 r = radeon_irq_kms_init(rdev);
@@ -7867,6 +7918,23 @@ static int cik_startup(struct radeon_device *rdev)
                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
         }
 
+        r = -ENOENT;
+
+        ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
+        if (ring->ring_size)
+                r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
+                                     VCE_CMD_NO_OP);
+
+        ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
+        if (ring->ring_size)
+                r = radeon_ring_init(rdev, ring, ring->ring_size, 0,
+                                     VCE_CMD_NO_OP);
+
+        if (!r)
+                r = vce_v1_0_init(rdev);
+        else if (r != -ENOENT)
+                DRM_ERROR("radeon: failed initializing VCE (%d).\n", r);
+
         r = radeon_ib_pool_init(rdev);
         if (r) {
                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
@@ -7938,6 +8006,7 @@ int cik_suspend(struct radeon_device *rdev)
         cik_sdma_enable(rdev, false);
         uvd_v1_0_fini(rdev);
         radeon_uvd_suspend(rdev);
+        radeon_vce_suspend(rdev);
         cik_fini_pg(rdev);
         cik_fini_cg(rdev);
         cik_irq_suspend(rdev);
@@ -8070,6 +8139,17 @@ int cik_init(struct radeon_device *rdev)
                 r600_ring_init(rdev, ring, 4096);
         }
 
+        r = radeon_vce_init(rdev);
+        if (!r) {
+                ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
+                ring->ring_obj = NULL;
+                r600_ring_init(rdev, ring, 4096);
+
+                ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
+                ring->ring_obj = NULL;
+                r600_ring_init(rdev, ring, 4096);
+        }
+
         rdev->ih.ring_obj = NULL;
         r600_ih_ring_init(rdev, 64 * 1024);
 
@@ -8131,6 +8211,7 @@ void cik_fini(struct radeon_device *rdev)
         radeon_irq_kms_fini(rdev);
         uvd_v1_0_fini(rdev);
         radeon_uvd_fini(rdev);
+        radeon_vce_fini(rdev);
         cik_pcie_gart_fini(rdev);
         r600_vram_scratch_fini(rdev);
         radeon_gem_fini(rdev);
@@ -8869,6 +8950,41 @@ int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
         return r;
 }
 
+int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk)
+{
+        int r, i;
+        struct atom_clock_dividers dividers;
+        u32 tmp;
+
+        r = radeon_atom_get_clock_dividers(rdev, COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
+                                           ecclk, false, &dividers);
+        if (r)
+                return r;
+
+        for (i = 0; i < 100; i++) {
+                if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
+                        break;
+                mdelay(10);
+        }
+        if (i == 100)
+                return -ETIMEDOUT;
+
+        tmp = RREG32_SMC(CG_ECLK_CNTL);
+        tmp &= ~(ECLK_DIR_CNTL_EN|ECLK_DIVIDER_MASK);
+        tmp |= dividers.post_divider;
+        WREG32_SMC(CG_ECLK_CNTL, tmp);
+
+        for (i = 0; i < 100; i++) {
+                if (RREG32_SMC(CG_ECLK_STATUS) & ECLK_STATUS)
+                        break;
+                mdelay(10);
+        }
+        if (i == 100)
+                return -ETIMEDOUT;
+
+        return 0;
+}
+
 static void cik_pcie_gen3_enable(struct radeon_device *rdev)
 {
         struct pci_dev *root = rdev->pdev->bus->self;
diff --git a/drivers/gpu/drm/radeon/cik_sdma.c b/drivers/gpu/drm/radeon/cik_sdma.c
index 94626ea90fa5..89b4afa5041c 100644
--- a/drivers/gpu/drm/radeon/cik_sdma.c
+++ b/drivers/gpu/drm/radeon/cik_sdma.c
@@ -369,8 +369,6 @@ static int cik_sdma_gfx_resume(struct radeon_device *rdev)
                 ring->wptr = 0;
                 WREG32(SDMA0_GFX_RB_WPTR + reg_offset, ring->wptr << 2);
 
-                ring->rptr = RREG32(SDMA0_GFX_RB_RPTR + reg_offset) >> 2;
-
                 /* enable DMA RB */
                 WREG32(SDMA0_GFX_RB_CNTL + reg_offset, rb_cntl | SDMA_RB_ENABLE);
 
@@ -713,11 +711,9 @@ bool cik_sdma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
                 mask = RADEON_RESET_DMA1;
 
         if (!(reset_mask & mask)) {
-                radeon_ring_lockup_update(ring);
+                radeon_ring_lockup_update(rdev, ring);
                 return false;
         }
-        /* force ring activities */
-        radeon_ring_force_activity(rdev, ring);
         return radeon_ring_test_lockup(rdev, ring);
 }
 
diff --git a/drivers/gpu/drm/radeon/cikd.h b/drivers/gpu/drm/radeon/cikd.h
index 98bae9d7b74d..213873270d5f 100644
--- a/drivers/gpu/drm/radeon/cikd.h
+++ b/drivers/gpu/drm/radeon/cikd.h
@@ -203,6 +203,12 @@
 #define         CTF_TEMP_MASK                           0x0003fe00
 #define         CTF_TEMP_SHIFT                          9
 
+#define CG_ECLK_CNTL                                    0xC05000AC
+#       define ECLK_DIVIDER_MASK                        0x7f
+#       define ECLK_DIR_CNTL_EN                         (1 << 8)
+#define CG_ECLK_STATUS                                  0xC05000B0
+#       define ECLK_STATUS                              (1 << 0)
+
 #define CG_SPLL_FUNC_CNTL                               0xC0500140
 #define         SPLL_RESET                              (1 << 0)
 #define         SPLL_PWRON                              (1 << 1)
@@ -2010,4 +2016,47 @@
 /* UVD CTX indirect */
 #define UVD_CGC_MEM_CTRL                                0xC0
 
+/* VCE */
+
+#define VCE_VCPU_CACHE_OFFSET0          0x20024
+#define VCE_VCPU_CACHE_SIZE0            0x20028
+#define VCE_VCPU_CACHE_OFFSET1          0x2002c
+#define VCE_VCPU_CACHE_SIZE1            0x20030
+#define VCE_VCPU_CACHE_OFFSET2          0x20034
+#define VCE_VCPU_CACHE_SIZE2            0x20038
+#define VCE_RB_RPTR2                    0x20178
+#define VCE_RB_WPTR2                    0x2017c
+#define VCE_RB_RPTR                     0x2018c
+#define VCE_RB_WPTR                     0x20190
+#define VCE_CLOCK_GATING_A              0x202f8
+#       define CGC_CLK_GATE_DLY_TIMER_MASK      (0xf << 0)
+#       define CGC_CLK_GATE_DLY_TIMER(x)        ((x) << 0)
+#       define CGC_CLK_GATER_OFF_DLY_TIMER_MASK (0xff << 4)
+#       define CGC_CLK_GATER_OFF_DLY_TIMER(x)   ((x) << 4)
+#       define CGC_UENC_WAIT_AWAKE      (1 << 18)
+#define VCE_CLOCK_GATING_B              0x202fc
+#define VCE_CGTT_CLK_OVERRIDE           0x207a0
+#define VCE_UENC_CLOCK_GATING           0x207bc
+#       define CLOCK_ON_DELAY_MASK      (0xf << 0)
+#       define CLOCK_ON_DELAY(x)        ((x) << 0)
+#       define CLOCK_OFF_DELAY_MASK     (0xff << 4)
+#       define CLOCK_OFF_DELAY(x)       ((x) << 4)
+#define VCE_UENC_REG_CLOCK_GATING       0x207c0
+#define VCE_SYS_INT_EN                  0x21300
+#       define VCE_SYS_INT_TRAP_INTERRUPT_EN    (1 << 3)
+#define VCE_LMI_CTRL2                   0x21474
+#define VCE_LMI_CTRL                    0x21498
+#define VCE_LMI_VM_CTRL                 0x214a0
+#define VCE_LMI_SWAP_CNTL               0x214b4
+#define VCE_LMI_SWAP_CNTL1              0x214b8
+#define VCE_LMI_CACHE_CTRL              0x214f4
+
+#define VCE_CMD_NO_OP           0x00000000
+#define VCE_CMD_END             0x00000001
+#define VCE_CMD_IB              0x00000002
+#define VCE_CMD_FENCE           0x00000003
+#define VCE_CMD_TRAP            0x00000004
+#define VCE_CMD_IB_AUTO         0x00000005
+#define VCE_CMD_SEMAPHORE       0x00000006
+
 #endif
diff --git a/drivers/gpu/drm/radeon/cypress_dpm.c b/drivers/gpu/drm/radeon/cypress_dpm.c
index cf783fc0ef21..5a9a5f4d7888 100644
--- a/drivers/gpu/drm/radeon/cypress_dpm.c
+++ b/drivers/gpu/drm/radeon/cypress_dpm.c
@@ -2036,6 +2036,10 @@ int cypress_dpm_init(struct radeon_device *rdev)
         pi->min_vddc_in_table = 0;
         pi->max_vddc_in_table = 0;
 
+        ret = r600_get_platform_caps(rdev);
+        if (ret)
+                return ret;
+
         ret = rv7xx_parse_power_table(rdev);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/radeon/evergreen.c b/drivers/gpu/drm/radeon/evergreen.c
index 27b0ff16082e..b406546440da 100644
--- a/drivers/gpu/drm/radeon/evergreen.c
+++ b/drivers/gpu/drm/radeon/evergreen.c
@@ -2990,8 +2990,6 @@ static int evergreen_cp_resume(struct radeon_device *rdev)
         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
 
-        ring->rptr = RREG32(CP_RB_RPTR);
-
         evergreen_cp_start(rdev);
         ring->ready = true;
         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
@@ -3952,11 +3950,9 @@ bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *rin
         if (!(reset_mask & (RADEON_RESET_GFX |
                             RADEON_RESET_COMPUTE |
                             RADEON_RESET_CP))) {
-                radeon_ring_lockup_update(ring);
+                radeon_ring_lockup_update(rdev, ring);
                 return false;
         }
-        /* force CP activities */
-        radeon_ring_force_activity(rdev, ring);
         return radeon_ring_test_lockup(rdev, ring);
 }
 
diff --git a/drivers/gpu/drm/radeon/evergreen_cs.c b/drivers/gpu/drm/radeon/evergreen_cs.c
index c7cac07f139b..5c8b358f9fba 100644
--- a/drivers/gpu/drm/radeon/evergreen_cs.c
+++ b/drivers/gpu/drm/radeon/evergreen_cs.c
@@ -1165,7 +1165,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                                         "0x%04X\n", reg);
                         return -EINVAL;
                 }
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 break;
         case DB_DEPTH_CONTROL:
                 track->db_depth_control = radeon_get_ib_value(p, idx);
@@ -1196,12 +1196,12 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         }
                         ib[idx] &= ~Z_ARRAY_MODE(0xf);
                         track->db_z_info &= ~Z_ARRAY_MODE(0xf);
-                        ib[idx] |= Z_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
-                        track->db_z_info |= Z_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) {
+                        ib[idx] |= Z_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags));
+                        track->db_z_info |= Z_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags));
+                        if (reloc->tiling_flags & RADEON_TILING_MACRO) {
                                 unsigned bankw, bankh, mtaspect, tile_split;
 
-                                evergreen_tiling_fields(reloc->lobj.tiling_flags,
+                                evergreen_tiling_fields(reloc->tiling_flags,
                                                         &bankw, &bankh, &mtaspect,
                                                         &tile_split);
                                 ib[idx] |= DB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks));
@@ -1237,7 +1237,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         return -EINVAL;
                 }
                 track->db_z_read_offset = radeon_get_ib_value(p, idx);
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 track->db_z_read_bo = reloc->robj;
                 track->db_dirty = true;
                 break;
@@ -1249,7 +1249,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         return -EINVAL;
                 }
                 track->db_z_write_offset = radeon_get_ib_value(p, idx);
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 track->db_z_write_bo = reloc->robj;
                 track->db_dirty = true;
                 break;
@@ -1261,7 +1261,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         return -EINVAL;
                 }
                 track->db_s_read_offset = radeon_get_ib_value(p, idx);
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 track->db_s_read_bo = reloc->robj;
                 track->db_dirty = true;
                 break;
@@ -1273,7 +1273,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         return -EINVAL;
                 }
                 track->db_s_write_offset = radeon_get_ib_value(p, idx);
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 track->db_s_write_bo = reloc->robj;
                 track->db_dirty = true;
                 break;
@@ -1297,7 +1297,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                 }
                 tmp = (reg - VGT_STRMOUT_BUFFER_BASE_0) / 16;
                 track->vgt_strmout_bo_offset[tmp] = radeon_get_ib_value(p, idx) << 8;
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 track->vgt_strmout_bo[tmp] = reloc->robj;
                 track->streamout_dirty = true;
                 break;
@@ -1317,7 +1317,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                                         "0x%04X\n", reg);
                         return -EINVAL;
                 }
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
         case CB_TARGET_MASK:
                 track->cb_target_mask = radeon_get_ib_value(p, idx);
                 track->cb_dirty = true;
@@ -1381,8 +1381,8 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                                                 "0x%04X\n", reg);
                                 return -EINVAL;
                         }
-                        ib[idx] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
-                        track->cb_color_info[tmp] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
+                        ib[idx] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags));
+                        track->cb_color_info[tmp] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags));
                 }
                 track->cb_dirty = true;
                 break;
@@ -1399,8 +1399,8 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                                                 "0x%04X\n", reg);
                                 return -EINVAL;
                         }
-                        ib[idx] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
-                        track->cb_color_info[tmp] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
+                        ib[idx] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags));
+                        track->cb_color_info[tmp] |= CB_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags));
                 }
                 track->cb_dirty = true;
                 break;
@@ -1461,10 +1461,10 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         return -EINVAL;
                 }
                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) {
+                        if (reloc->tiling_flags & RADEON_TILING_MACRO) {
                                 unsigned bankw, bankh, mtaspect, tile_split;
 
-                                evergreen_tiling_fields(reloc->lobj.tiling_flags,
+                                evergreen_tiling_fields(reloc->tiling_flags,
                                                         &bankw, &bankh, &mtaspect,
                                                         &tile_split);
                                 ib[idx] |= CB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks));
@@ -1489,10 +1489,10 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         return -EINVAL;
                 }
                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) {
+                        if (reloc->tiling_flags & RADEON_TILING_MACRO) {
                                 unsigned bankw, bankh, mtaspect, tile_split;
 
-                                evergreen_tiling_fields(reloc->lobj.tiling_flags,
+                                evergreen_tiling_fields(reloc->tiling_flags,
                                                         &bankw, &bankh, &mtaspect,
                                                         &tile_split);
                                 ib[idx] |= CB_NUM_BANKS(evergreen_cs_get_num_banks(track->nbanks));
@@ -1520,7 +1520,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         dev_err(p->dev, "bad SET_CONTEXT_REG 0x%04X\n", reg);
                         return -EINVAL;
                 }
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 track->cb_color_fmask_bo[tmp] = reloc->robj;
                 break;
         case CB_COLOR0_CMASK:
@@ -1537,7 +1537,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         dev_err(p->dev, "bad SET_CONTEXT_REG 0x%04X\n", reg);
                         return -EINVAL;
                 }
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 track->cb_color_cmask_bo[tmp] = reloc->robj;
                 break;
         case CB_COLOR0_FMASK_SLICE:
@@ -1578,7 +1578,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                 }
                 tmp = (reg - CB_COLOR0_BASE) / 0x3c;
                 track->cb_color_bo_offset[tmp] = radeon_get_ib_value(p, idx);
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 track->cb_color_bo[tmp] = reloc->robj;
                 track->cb_dirty = true;
                 break;
@@ -1594,7 +1594,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                 }
                 tmp = ((reg - CB_COLOR8_BASE) / 0x1c) + 8;
                 track->cb_color_bo_offset[tmp] = radeon_get_ib_value(p, idx);
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 track->cb_color_bo[tmp] = reloc->robj;
                 track->cb_dirty = true;
                 break;
@@ -1606,7 +1606,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         return -EINVAL;
                 }
                 track->htile_offset = radeon_get_ib_value(p, idx);
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 track->htile_bo = reloc->robj;
                 track->db_dirty = true;
                 break;
@@ -1723,7 +1723,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                                         "0x%04X\n", reg);
                         return -EINVAL;
                 }
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 break;
         case SX_MEMORY_EXPORT_BASE:
                 if (p->rdev->family >= CHIP_CAYMAN) {
@@ -1737,7 +1737,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                                         "0x%04X\n", reg);
                         return -EINVAL;
                 }
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 break;
         case CAYMAN_SX_SCATTER_EXPORT_BASE:
                 if (p->rdev->family < CHIP_CAYMAN) {
@@ -1751,7 +1751,7 @@ static int evergreen_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                                         "0x%04X\n", reg);
                         return -EINVAL;
                 }
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 break;
         case SX_MISC:
                 track->sx_misc_kill_all_prims = (radeon_get_ib_value(p, idx) & 0x1) != 0;
@@ -1836,7 +1836,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                         return -EINVAL;
                 }
 
-                offset = reloc->lobj.gpu_offset +
+                offset = reloc->gpu_offset +
                          (idx_value & 0xfffffff0) +
                          ((u64)(tmp & 0xff) << 32);
 
@@ -1882,7 +1882,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                         return -EINVAL;
                 }
 
-                offset = reloc->lobj.gpu_offset +
+                offset = reloc->gpu_offset +
                          idx_value +
                          ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32);
 
@@ -1909,7 +1909,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                         return -EINVAL;
                 }
 
-                offset = reloc->lobj.gpu_offset +
+                offset = reloc->gpu_offset +
                          idx_value +
                          ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32);
 
@@ -1937,7 +1937,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                         return -EINVAL;
                 }
 
-                offset = reloc->lobj.gpu_offset +
+                offset = reloc->gpu_offset +
                          radeon_get_ib_value(p, idx+1) +
                          ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
 
@@ -2027,7 +2027,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                         DRM_ERROR("bad DISPATCH_INDIRECT\n");
                         return -EINVAL;
                 }
-                ib[idx+0] = idx_value + (u32)(reloc->lobj.gpu_offset & 0xffffffff);
+                ib[idx+0] = idx_value + (u32)(reloc->gpu_offset & 0xffffffff);
                 r = evergreen_cs_track_check(p);
                 if (r) {
                         dev_warn(p->dev, "%s:%d invalid cmd stream\n", __func__, __LINE__);
@@ -2049,7 +2049,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                                 return -EINVAL;
                         }
 
-                        offset = reloc->lobj.gpu_offset +
+                        offset = reloc->gpu_offset +
                                  (radeon_get_ib_value(p, idx+1) & 0xfffffffc) +
                                  ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
 
@@ -2106,7 +2106,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                                 tmp = radeon_get_ib_value(p, idx) +
                                         ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32);
 
-                                offset = reloc->lobj.gpu_offset + tmp;
+                                offset = reloc->gpu_offset + tmp;
 
                                 if ((tmp + size) > radeon_bo_size(reloc->robj)) {
                                         dev_warn(p->dev, "CP DMA src buffer too small (%llu %lu)\n",
@@ -2144,7 +2144,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                                 tmp = radeon_get_ib_value(p, idx+2) +
                                         ((u64)(radeon_get_ib_value(p, idx+3) & 0xff) << 32);
 
-                                offset = reloc->lobj.gpu_offset + tmp;
+                                offset = reloc->gpu_offset + tmp;
 
                                 if ((tmp + size) > radeon_bo_size(reloc->robj)) {
                                         dev_warn(p->dev, "CP DMA dst buffer too small (%llu %lu)\n",
@@ -2174,7 +2174,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                                 DRM_ERROR("bad SURFACE_SYNC\n");
                                 return -EINVAL;
                         }
-                        ib[idx+2] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                        ib[idx+2] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 }
                 break;
         case PACKET3_EVENT_WRITE:
@@ -2190,7 +2190,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                                 DRM_ERROR("bad EVENT_WRITE\n");
                                 return -EINVAL;
                         }
-                        offset = reloc->lobj.gpu_offset +
+                        offset = reloc->gpu_offset +
                                  (radeon_get_ib_value(p, idx+1) & 0xfffffff8) +
                                  ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
 
@@ -2212,7 +2212,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                         return -EINVAL;
                 }
 
-                offset = reloc->lobj.gpu_offset +
+                offset = reloc->gpu_offset +
                          (radeon_get_ib_value(p, idx+1) & 0xfffffffc) +
                          ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
 
@@ -2234,7 +2234,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                         return -EINVAL;
                 }
 
-                offset = reloc->lobj.gpu_offset +
+                offset = reloc->gpu_offset +
                          (radeon_get_ib_value(p, idx+1) & 0xfffffffc) +
                          ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
 
@@ -2302,11 +2302,11 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                                 }
                                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
                                         ib[idx+1+(i*8)+1] |=
-                                                TEX_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->lobj.tiling_flags));
-                                        if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) {
+                                                TEX_ARRAY_MODE(evergreen_cs_get_aray_mode(reloc->tiling_flags));
+                                        if (reloc->tiling_flags & RADEON_TILING_MACRO) {
                                                 unsigned bankw, bankh, mtaspect, tile_split;
 
-                                                evergreen_tiling_fields(reloc->lobj.tiling_flags,
+                                                evergreen_tiling_fields(reloc->tiling_flags,
                                                                         &bankw, &bankh, &mtaspect,
                                                                         &tile_split);
                                                 ib[idx+1+(i*8)+6] |= TEX_TILE_SPLIT(tile_split);
@@ -2318,7 +2318,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                                         }
                                 }
                                 texture = reloc->robj;
-                                toffset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                                toffset = (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
 
                                 /* tex mip base */
                                 tex_dim = ib[idx+1+(i*8)+0] & 0x7;
@@ -2337,7 +2337,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                                                 DRM_ERROR("bad SET_RESOURCE (tex)\n");
                                                 return -EINVAL;
                                         }
-                                        moffset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                                        moffset = (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                                         mipmap = reloc->robj;
                                 }
 
@@ -2364,7 +2364,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                                         ib[idx+1+(i*8)+1] = radeon_bo_size(reloc->robj) - offset;
                                 }
 
-                                offset64 = reloc->lobj.gpu_offset + offset;
+                                offset64 = reloc->gpu_offset + offset;
                                 ib[idx+1+(i*8)+0] = offset64;
                                 ib[idx+1+(i*8)+2] = (ib[idx+1+(i*8)+2] & 0xffffff00) |
                                                     (upper_32_bits(offset64) & 0xff);
@@ -2445,7 +2445,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                                           offset + 4, radeon_bo_size(reloc->robj));
                                 return -EINVAL;
                         }
-                        offset += reloc->lobj.gpu_offset;
+                        offset += reloc->gpu_offset;
                         ib[idx+1] = offset;
                         ib[idx+2] = upper_32_bits(offset) & 0xff;
                 }
@@ -2464,7 +2464,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                                           offset + 4, radeon_bo_size(reloc->robj));
                                 return -EINVAL;
                         }
-                        offset += reloc->lobj.gpu_offset;
+                        offset += reloc->gpu_offset;
                         ib[idx+3] = offset;
                         ib[idx+4] = upper_32_bits(offset) & 0xff;
                 }
@@ -2493,7 +2493,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                                   offset + 8, radeon_bo_size(reloc->robj));
                         return -EINVAL;
                 }
-                offset += reloc->lobj.gpu_offset;
+                offset += reloc->gpu_offset;
                 ib[idx+0] = offset;
                 ib[idx+1] = upper_32_bits(offset) & 0xff;
                 break;
@@ -2518,7 +2518,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                                           offset + 4, radeon_bo_size(reloc->robj));
                                 return -EINVAL;
                         }
-                        offset += reloc->lobj.gpu_offset;
+                        offset += reloc->gpu_offset;
                         ib[idx+1] = offset;
                         ib[idx+2] = upper_32_bits(offset) & 0xff;
                 } else {
@@ -2542,7 +2542,7 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                                           offset + 4, radeon_bo_size(reloc->robj));
                                 return -EINVAL;
                         }
-                        offset += reloc->lobj.gpu_offset;
+                        offset += reloc->gpu_offset;
                         ib[idx+3] = offset;
                         ib[idx+4] = upper_32_bits(offset) & 0xff;
                 } else {
@@ -2717,7 +2717,7 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
                                 dst_offset = radeon_get_ib_value(p, idx+1);
                                 dst_offset <<= 8;
 
-                                ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
+                                ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8);
                                 p->idx += count + 7;
                                 break;
                         /* linear */
@@ -2725,8 +2725,8 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
                                 dst_offset = radeon_get_ib_value(p, idx+1);
                                 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+2) & 0xff)) << 32;
 
-                                ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
-                                ib[idx+2] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
+                                ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc);
+                                ib[idx+2] += upper_32_bits(dst_reloc->gpu_offset) & 0xff;
                                 p->idx += count + 3;
                                 break;
                         default:
@@ -2768,10 +2768,10 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
                                                         dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
                                         return -EINVAL;
                                 }
-                                ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
-                                ib[idx+2] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
-                                ib[idx+3] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
-                                ib[idx+4] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
+                                ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc);
+                                ib[idx+2] += (u32)(src_reloc->gpu_offset & 0xfffffffc);
+                                ib[idx+3] += upper_32_bits(dst_reloc->gpu_offset) & 0xff;
+                                ib[idx+4] += upper_32_bits(src_reloc->gpu_offset) & 0xff;
                                 p->idx += 5;
                                 break;
                         /* Copy L2T/T2L */
@@ -2781,22 +2781,22 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
                                         /* tiled src, linear dst */
                                         src_offset = radeon_get_ib_value(p, idx+1);
                                         src_offset <<= 8;
-                                        ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8);
+                                        ib[idx+1] += (u32)(src_reloc->gpu_offset >> 8);
 
                                         dst_offset = radeon_get_ib_value(p, idx + 7);
                                         dst_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32;
-                                        ib[idx+7] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
-                                        ib[idx+8] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
+                                        ib[idx+7] += (u32)(dst_reloc->gpu_offset & 0xfffffffc);
+                                        ib[idx+8] += upper_32_bits(dst_reloc->gpu_offset) & 0xff;
                                 } else {
                                         /* linear src, tiled dst */
                                         src_offset = radeon_get_ib_value(p, idx+7);
                                         src_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32;
-                                        ib[idx+7] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
-                                        ib[idx+8] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
+                                        ib[idx+7] += (u32)(src_reloc->gpu_offset & 0xfffffffc);
+                                        ib[idx+8] += upper_32_bits(src_reloc->gpu_offset) & 0xff;
 
                                         dst_offset = radeon_get_ib_value(p, idx+1);
                                         dst_offset <<= 8;
-                                        ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
+                                        ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8);
                                 }
                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
                                         dev_warn(p->dev, "DMA L2T, src buffer too small (%llu %lu)\n",
@@ -2827,10 +2827,10 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
                                                         dst_offset + count, radeon_bo_size(dst_reloc->robj));
                                         return -EINVAL;
                                 }
-                                ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xffffffff);
-                                ib[idx+2] += (u32)(src_reloc->lobj.gpu_offset & 0xffffffff);
-                                ib[idx+3] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
-                                ib[idx+4] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
+                                ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xffffffff);
+                                ib[idx+2] += (u32)(src_reloc->gpu_offset & 0xffffffff);
+                                ib[idx+3] += upper_32_bits(dst_reloc->gpu_offset) & 0xff;
+                                ib[idx+4] += upper_32_bits(src_reloc->gpu_offset) & 0xff;
                                 p->idx += 5;
                                 break;
                         /* Copy L2L, partial */
@@ -2840,10 +2840,10 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
                                         DRM_ERROR("L2L Partial is cayman only !\n");
                                         return -EINVAL;
                                 }
-                                ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset & 0xffffffff);
-                                ib[idx+2] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
-                                ib[idx+4] += (u32)(dst_reloc->lobj.gpu_offset & 0xffffffff);
-                                ib[idx+5] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
+                                ib[idx+1] += (u32)(src_reloc->gpu_offset & 0xffffffff);
+                                ib[idx+2] += upper_32_bits(src_reloc->gpu_offset) & 0xff;
+                                ib[idx+4] += (u32)(dst_reloc->gpu_offset & 0xffffffff);
+                                ib[idx+5] += upper_32_bits(dst_reloc->gpu_offset) & 0xff;
 
                                 p->idx += 9;
                                 break;
@@ -2876,12 +2876,12 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
                                                         dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj));
                                         return -EINVAL;
                                 }
-                                ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
-                                ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset & 0xfffffffc);
-                                ib[idx+3] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
-                                ib[idx+4] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
-                                ib[idx+5] += upper_32_bits(dst2_reloc->lobj.gpu_offset) & 0xff;
-                                ib[idx+6] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
+                                ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc);
+                                ib[idx+2] += (u32)(dst2_reloc->gpu_offset & 0xfffffffc);
+                                ib[idx+3] += (u32)(src_reloc->gpu_offset & 0xfffffffc);
+                                ib[idx+4] += upper_32_bits(dst_reloc->gpu_offset) & 0xff;
+                                ib[idx+5] += upper_32_bits(dst2_reloc->gpu_offset) & 0xff;
+                                ib[idx+6] += upper_32_bits(src_reloc->gpu_offset) & 0xff;
                                 p->idx += 7;
                                 break;
                         /* Copy L2T Frame to Field */
@@ -2916,10 +2916,10 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
                                                         dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj));
                                         return -EINVAL;
                                 }
-                                ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
-                                ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset >> 8);
-                                ib[idx+8] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
-                                ib[idx+9] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
+                                ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8);
+                                ib[idx+2] += (u32)(dst2_reloc->gpu_offset >> 8);
+                                ib[idx+8] += (u32)(src_reloc->gpu_offset & 0xfffffffc);
+                                ib[idx+9] += upper_32_bits(src_reloc->gpu_offset) & 0xff;
                                 p->idx += 10;
                                 break;
                         /* Copy L2T/T2L, partial */
@@ -2932,16 +2932,16 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
                                 /* detile bit */
                                 if (radeon_get_ib_value(p, idx + 2) & (1 << 31)) {
                                         /* tiled src, linear dst */
-                                        ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8);
+                                        ib[idx+1] += (u32)(src_reloc->gpu_offset >> 8);
 
-                                        ib[idx+7] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
-                                        ib[idx+8] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
+                                        ib[idx+7] += (u32)(dst_reloc->gpu_offset & 0xfffffffc);
+                                        ib[idx+8] += upper_32_bits(dst_reloc->gpu_offset) & 0xff;
                                 } else {
                                         /* linear src, tiled dst */
-                                        ib[idx+7] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
-                                        ib[idx+8] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
+                                        ib[idx+7] += (u32)(src_reloc->gpu_offset & 0xfffffffc);
+                                        ib[idx+8] += upper_32_bits(src_reloc->gpu_offset) & 0xff;
 
-                                        ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
+                                        ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8);
                                 }
                                 p->idx += 12;
                                 break;
@@ -2978,10 +2978,10 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
                                                         dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj));
                                         return -EINVAL;
                                 }
-                                ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
-                                ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset >> 8);
-                                ib[idx+8] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
-                                ib[idx+9] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
+                                ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8);
+                                ib[idx+2] += (u32)(dst2_reloc->gpu_offset >> 8);
+                                ib[idx+8] += (u32)(src_reloc->gpu_offset & 0xfffffffc);
+                                ib[idx+9] += upper_32_bits(src_reloc->gpu_offset) & 0xff;
                                 p->idx += 10;
                                 break;
                         /* Copy L2T/T2L (tile units) */
@@ -2992,22 +2992,22 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
                                         /* tiled src, linear dst */
                                         src_offset = radeon_get_ib_value(p, idx+1);
                                         src_offset <<= 8;
-                                        ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8);
+                                        ib[idx+1] += (u32)(src_reloc->gpu_offset >> 8);
 
                                         dst_offset = radeon_get_ib_value(p, idx+7);
                                         dst_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32;
-                                        ib[idx+7] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
-                                        ib[idx+8] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
+                                        ib[idx+7] += (u32)(dst_reloc->gpu_offset & 0xfffffffc);
+                                        ib[idx+8] += upper_32_bits(dst_reloc->gpu_offset) & 0xff;
                                 } else {
                                         /* linear src, tiled dst */
                                         src_offset = radeon_get_ib_value(p, idx+7);
                                         src_offset |= ((u64)(radeon_get_ib_value(p, idx+8) & 0xff)) << 32;
-                                        ib[idx+7] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
-                                        ib[idx+8] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
+                                        ib[idx+7] += (u32)(src_reloc->gpu_offset & 0xfffffffc);
+                                        ib[idx+8] += upper_32_bits(src_reloc->gpu_offset) & 0xff;
 
                                         dst_offset = radeon_get_ib_value(p, idx+1);
                                         dst_offset <<= 8;
-                                        ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
+                                        ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8);
                                 }
                                 if ((src_offset + (count * 4)) > radeon_bo_size(src_reloc->robj)) {
                                         dev_warn(p->dev, "DMA L2T, T2L src buffer too small (%llu %lu)\n",
@@ -3028,8 +3028,8 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
                                         DRM_ERROR("L2T, T2L Partial is cayman only !\n");
                                         return -EINVAL;
                                 }
-                                ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8);
-                                ib[idx+4] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
+                                ib[idx+1] += (u32)(src_reloc->gpu_offset >> 8);
+                                ib[idx+4] += (u32)(dst_reloc->gpu_offset >> 8);
                                 p->idx += 13;
                                 break;
                         /* Copy L2T broadcast (tile units) */
@@ -3065,10 +3065,10 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
                                                         dst2_offset + (count * 4), radeon_bo_size(dst2_reloc->robj));
                                         return -EINVAL;
                                 }
-                                ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
-                                ib[idx+2] += (u32)(dst2_reloc->lobj.gpu_offset >> 8);
-                                ib[idx+8] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
-                                ib[idx+9] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
+                                ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8);
+                                ib[idx+2] += (u32)(dst2_reloc->gpu_offset >> 8);
+                                ib[idx+8] += (u32)(src_reloc->gpu_offset & 0xfffffffc);
+                                ib[idx+9] += upper_32_bits(src_reloc->gpu_offset) & 0xff;
                                 p->idx += 10;
                                 break;
                         default:
@@ -3089,8 +3089,8 @@ int evergreen_dma_cs_parse(struct radeon_cs_parser *p)
                                          dst_offset, radeon_bo_size(dst_reloc->robj));
                                 return -EINVAL;
                         }
-                        ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
-                        ib[idx+3] += (upper_32_bits(dst_reloc->lobj.gpu_offset) << 16) & 0x00ff0000;
+                        ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc);
+                        ib[idx+3] += (upper_32_bits(dst_reloc->gpu_offset) << 16) & 0x00ff0000;
                         p->idx += 4;
                         break;
                 case DMA_PACKET_NOP:
diff --git a/drivers/gpu/drm/radeon/evergreen_dma.c b/drivers/gpu/drm/radeon/evergreen_dma.c
index a37b54436382..287fe966d7de 100644
--- a/drivers/gpu/drm/radeon/evergreen_dma.c
+++ b/drivers/gpu/drm/radeon/evergreen_dma.c
@@ -174,11 +174,9 @@ bool evergreen_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *rin
         u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
 
         if (!(reset_mask & RADEON_RESET_DMA)) {
-                radeon_ring_lockup_update(ring);
+                radeon_ring_lockup_update(rdev, ring);
                 return false;
         }
-        /* force ring activities */
-        radeon_ring_force_activity(rdev, ring);
         return radeon_ring_test_lockup(rdev, ring);
 }
 
diff --git a/drivers/gpu/drm/radeon/kv_dpm.c b/drivers/gpu/drm/radeon/kv_dpm.c
index 351db361239d..16ec9d56a234 100644
--- a/drivers/gpu/drm/radeon/kv_dpm.c
+++ b/drivers/gpu/drm/radeon/kv_dpm.c
@@ -1338,13 +1338,11 @@ static int kv_enable_uvd_dpm(struct radeon_device *rdev, bool enable)
                                         PPSMC_MSG_UVDDPM_Enable : PPSMC_MSG_UVDDPM_Disable);
 }
 
-#if 0
 static int kv_enable_vce_dpm(struct radeon_device *rdev, bool enable)
 {
         return kv_notify_message_to_smu(rdev, enable ?
                                         PPSMC_MSG_VCEDPM_Enable : PPSMC_MSG_VCEDPM_Disable);
 }
-#endif
 
 static int kv_enable_samu_dpm(struct radeon_device *rdev, bool enable)
 {
@@ -1389,7 +1387,6 @@ static int kv_update_uvd_dpm(struct radeon_device *rdev, bool gate)
         return kv_enable_uvd_dpm(rdev, !gate);
 }
 
-#if 0
 static u8 kv_get_vce_boot_level(struct radeon_device *rdev)
 {
         u8 i;
@@ -1414,6 +1411,9 @@ static int kv_update_vce_dpm(struct radeon_device *rdev,
         int ret;
 
         if (radeon_new_state->evclk > 0 && radeon_current_state->evclk == 0) {
+                kv_dpm_powergate_vce(rdev, false);
+                /* turn the clocks on when encoding */
+                cik_update_cg(rdev, RADEON_CG_BLOCK_VCE, false);
                 if (pi->caps_stable_p_state)
                         pi->vce_boot_level = table->count - 1;
                 else
@@ -1436,11 +1436,13 @@ static int kv_update_vce_dpm(struct radeon_device *rdev,
                 kv_enable_vce_dpm(rdev, true);
         } else if (radeon_new_state->evclk == 0 && radeon_current_state->evclk > 0) {
                 kv_enable_vce_dpm(rdev, false);
+                /* turn the clocks off when not encoding */
+                cik_update_cg(rdev, RADEON_CG_BLOCK_VCE, true);
+                kv_dpm_powergate_vce(rdev, true);
         }
 
         return 0;
 }
-#endif
 
 static int kv_update_samu_dpm(struct radeon_device *rdev, bool gate)
 {
@@ -1575,11 +1577,16 @@ static void kv_dpm_powergate_vce(struct radeon_device *rdev, bool gate)
         pi->vce_power_gated = gate;
 
         if (gate) {
-                if (pi->caps_vce_pg)
+                if (pi->caps_vce_pg) {
+                        /* XXX do we need a vce_v1_0_stop() ?  */
                         kv_notify_message_to_smu(rdev, PPSMC_MSG_VCEPowerOFF);
+                }
         } else {
-                if (pi->caps_vce_pg)
+                if (pi->caps_vce_pg) {
                         kv_notify_message_to_smu(rdev, PPSMC_MSG_VCEPowerON);
+                        vce_v2_0_resume(rdev);
+                        vce_v1_0_start(rdev);
+                }
         }
 }
 
@@ -1768,7 +1775,7 @@ int kv_dpm_set_power_state(struct radeon_device *rdev)
 {
         struct kv_power_info *pi = kv_get_pi(rdev);
         struct radeon_ps *new_ps = &pi->requested_rps;
-        /*struct radeon_ps *old_ps = &pi->current_rps;*/
+        struct radeon_ps *old_ps = &pi->current_rps;
         int ret;
 
         if (pi->bapm_enable) {
@@ -1798,13 +1805,12 @@ int kv_dpm_set_power_state(struct radeon_device *rdev)
                         kv_set_enabled_levels(rdev);
                         kv_force_lowest_valid(rdev);
                         kv_unforce_levels(rdev);
-#if 0
+
                         ret = kv_update_vce_dpm(rdev, new_ps, old_ps);
                         if (ret) {
                                 DRM_ERROR("kv_update_vce_dpm failed\n");
                                 return ret;
                         }
-#endif
                         kv_update_sclk_t(rdev);
                 }
         } else {
@@ -1823,13 +1829,11 @@ int kv_dpm_set_power_state(struct radeon_device *rdev)
                         kv_program_nbps_index_settings(rdev, new_ps);
                         kv_freeze_sclk_dpm(rdev, false);
                         kv_set_enabled_levels(rdev);
-#if 0
                         ret = kv_update_vce_dpm(rdev, new_ps, old_ps);
                         if (ret) {
                                 DRM_ERROR("kv_update_vce_dpm failed\n");
                                 return ret;
                         }
-#endif
                         kv_update_acp_boot_level(rdev);
                         kv_update_sclk_t(rdev);
                         kv_enable_nb_dpm(rdev);
@@ -2037,6 +2041,14 @@ static void kv_apply_state_adjust_rules(struct radeon_device *rdev,
         struct radeon_clock_and_voltage_limits *max_limits =
                 &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
 
+        if (new_rps->vce_active) {
+                new_rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk;
+                new_rps->ecclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].ecclk;
+        } else {
+                new_rps->evclk = 0;
+                new_rps->ecclk = 0;
+        }
+
         mclk = max_limits->mclk;
         sclk = min_sclk;
 
@@ -2056,6 +2068,11 @@ static void kv_apply_state_adjust_rules(struct radeon_device *rdev,
                 sclk = stable_p_state_sclk;
         }
 
+        if (new_rps->vce_active) {
+                if (sclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk)
+                        sclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk;
+        }
+
         ps->need_dfs_bypass = true;
 
         for (i = 0; i < ps->num_levels; i++) {
@@ -2092,7 +2109,8 @@ static void kv_apply_state_adjust_rules(struct radeon_device *rdev,
                 }
         }
 
-        pi->video_start = new_rps->dclk || new_rps->vclk;
+        pi->video_start = new_rps->dclk || new_rps->vclk ||
+                new_rps->evclk || new_rps->ecclk;
 
         if ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
             ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)
@@ -2538,9 +2556,6 @@ static int kv_parse_power_table(struct radeon_device *rdev)
         if (!rdev->pm.dpm.ps)
                 return -ENOMEM;
         power_state_offset = (u8 *)state_array->states;
-        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
-        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
-        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
         for (i = 0; i < state_array->ucNumEntries; i++) {
                 u8 *idx;
                 power_state = (union pplib_power_state *)power_state_offset;
@@ -2577,6 +2592,19 @@ static int kv_parse_power_table(struct radeon_device *rdev)
                 power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
         }
         rdev->pm.dpm.num_ps = state_array->ucNumEntries;
+
+        /* fill in the vce power states */
+        for (i = 0; i < RADEON_MAX_VCE_LEVELS; i++) {
+                u32 sclk;
+                clock_array_index = rdev->pm.dpm.vce_states[i].clk_idx;
+                clock_info = (union pplib_clock_info *)
+                        &clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
+                sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
+                sclk |= clock_info->sumo.ucEngineClockHigh << 16;
+                rdev->pm.dpm.vce_states[i].sclk = sclk;
+                rdev->pm.dpm.vce_states[i].mclk = 0;
+        }
+
         return 0;
 }
 
@@ -2590,6 +2618,10 @@ int kv_dpm_init(struct radeon_device *rdev)
                 return -ENOMEM;
         rdev->pm.dpm.priv = pi;
 
+        ret = r600_get_platform_caps(rdev);
+        if (ret)
+                return ret;
+
         ret = r600_parse_extended_power_table(rdev);
         if (ret)
                 return ret;
@@ -2623,7 +2655,7 @@ int kv_dpm_init(struct radeon_device *rdev)
         pi->caps_fps = false; /* true? */
         pi->caps_uvd_pg = true;
         pi->caps_uvd_dpm = true;
-        pi->caps_vce_pg = false;
+        pi->caps_vce_pg = false; /* XXX true */
         pi->caps_samu_pg = false;
         pi->caps_acp_pg = false;
         pi->caps_stable_p_state = false;
diff --git a/drivers/gpu/drm/radeon/ni.c b/drivers/gpu/drm/radeon/ni.c
index bf6300cfd62d..d246e043421a 100644
--- a/drivers/gpu/drm/radeon/ni.c
+++ b/drivers/gpu/drm/radeon/ni.c
@@ -1642,8 +1642,8 @@ static int cayman_cp_resume(struct radeon_device *rdev)
                 ring = &rdev->ring[ridx[i]];
                 WREG32_P(cp_rb_cntl[i], RB_RPTR_WR_ENA, ~RB_RPTR_WR_ENA);
 
-                ring->rptr = ring->wptr = 0;
-                WREG32(cp_rb_rptr[i], ring->rptr);
+                ring->wptr = 0;
+                WREG32(cp_rb_rptr[i], 0);
                 WREG32(cp_rb_wptr[i], ring->wptr);
 
                 mdelay(1);
@@ -1917,11 +1917,9 @@ bool cayman_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
         if (!(reset_mask & (RADEON_RESET_GFX |
                             RADEON_RESET_COMPUTE |
                             RADEON_RESET_CP))) {
-                radeon_ring_lockup_update(ring);
+                radeon_ring_lockup_update(rdev, ring);
                 return false;
         }
-        /* force CP activities */
-        radeon_ring_force_activity(rdev, ring);
         return radeon_ring_test_lockup(rdev, ring);
 }
 
diff --git a/drivers/gpu/drm/radeon/ni_dma.c b/drivers/gpu/drm/radeon/ni_dma.c
index 7cf96b15377f..6378e0276691 100644
--- a/drivers/gpu/drm/radeon/ni_dma.c
+++ b/drivers/gpu/drm/radeon/ni_dma.c
@@ -248,8 +248,6 @@ int cayman_dma_resume(struct radeon_device *rdev)
                 ring->wptr = 0;
                 WREG32(DMA_RB_WPTR + reg_offset, ring->wptr << 2);
 
-                ring->rptr = RREG32(DMA_RB_RPTR + reg_offset) >> 2;
-
                 WREG32(DMA_RB_CNTL + reg_offset, rb_cntl | DMA_RB_ENABLE);
 
                 ring->ready = true;
@@ -302,11 +300,9 @@ bool cayman_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
                 mask = RADEON_RESET_DMA1;
 
         if (!(reset_mask & mask)) {
-                radeon_ring_lockup_update(ring);
+                radeon_ring_lockup_update(rdev, ring);
                 return false;
         }
-        /* force ring activities */
-        radeon_ring_force_activity(rdev, ring);
         return radeon_ring_test_lockup(rdev, ring);
 }
 
diff --git a/drivers/gpu/drm/radeon/ni_dpm.c b/drivers/gpu/drm/radeon/ni_dpm.c
index ca814276b075..004c931606c4 100644
--- a/drivers/gpu/drm/radeon/ni_dpm.c
+++ b/drivers/gpu/drm/radeon/ni_dpm.c
@@ -4025,9 +4025,6 @@ static int ni_parse_power_table(struct radeon_device *rdev)
                                   power_info->pplib.ucNumStates, GFP_KERNEL);
         if (!rdev->pm.dpm.ps)
                 return -ENOMEM;
-        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
-        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
-        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
 
         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
                 power_state = (union pplib_power_state *)
@@ -4089,6 +4086,10 @@ int ni_dpm_init(struct radeon_device *rdev)
         pi->min_vddc_in_table = 0;
         pi->max_vddc_in_table = 0;
 
+        ret = r600_get_platform_caps(rdev);
+        if (ret)
+                return ret;
+
         ret = ni_parse_power_table(rdev);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/radeon/r100.c b/drivers/gpu/drm/radeon/r100.c
index 3cc78bb66042..b6c32640df20 100644
--- a/drivers/gpu/drm/radeon/r100.c
+++ b/drivers/gpu/drm/radeon/r100.c
@@ -1193,7 +1193,6 @@ int r100_cp_init(struct radeon_device *rdev, unsigned ring_size)
 
         WREG32(RADEON_CP_RB_CNTL, tmp);
         udelay(10);
-        ring->rptr = RREG32(RADEON_CP_RB_RPTR);
         /* Set cp mode to bus mastering & enable cp*/
         WREG32(RADEON_CP_CSQ_MODE,
                REG_SET(RADEON_INDIRECT2_START, indirect2_start) |
@@ -1275,12 +1274,12 @@ int r100_reloc_pitch_offset(struct radeon_cs_parser *p,
 
         value = radeon_get_ib_value(p, idx);
         tmp = value & 0x003fffff;
-        tmp += (((u32)reloc->lobj.gpu_offset) >> 10);
+        tmp += (((u32)reloc->gpu_offset) >> 10);
 
         if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
-                if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO)
+                if (reloc->tiling_flags & RADEON_TILING_MACRO)
                         tile_flags |= RADEON_DST_TILE_MACRO;
-                if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO) {
+                if (reloc->tiling_flags & RADEON_TILING_MICRO) {
                         if (reg == RADEON_SRC_PITCH_OFFSET) {
                                 DRM_ERROR("Cannot src blit from microtiled surface\n");
                                 radeon_cs_dump_packet(p, pkt);
@@ -1326,7 +1325,7 @@ int r100_packet3_load_vbpntr(struct radeon_cs_parser *p,
                         return r;
                 }
                 idx_value = radeon_get_ib_value(p, idx);
-                ib[idx+1] = radeon_get_ib_value(p, idx + 1) + ((u32)reloc->lobj.gpu_offset);
+                ib[idx+1] = radeon_get_ib_value(p, idx + 1) + ((u32)reloc->gpu_offset);
 
                 track->arrays[i + 0].esize = idx_value >> 8;
                 track->arrays[i + 0].robj = reloc->robj;
@@ -1338,7 +1337,7 @@ int r100_packet3_load_vbpntr(struct radeon_cs_parser *p,
                         radeon_cs_dump_packet(p, pkt);
                         return r;
                 }
-                ib[idx+2] = radeon_get_ib_value(p, idx + 2) + ((u32)reloc->lobj.gpu_offset);
+                ib[idx+2] = radeon_get_ib_value(p, idx + 2) + ((u32)reloc->gpu_offset);
                 track->arrays[i + 1].robj = reloc->robj;
                 track->arrays[i + 1].esize = idx_value >> 24;
                 track->arrays[i + 1].esize &= 0x7F;
@@ -1352,7 +1351,7 @@ int r100_packet3_load_vbpntr(struct radeon_cs_parser *p,
                         return r;
                 }
                 idx_value = radeon_get_ib_value(p, idx);
-                ib[idx+1] = radeon_get_ib_value(p, idx + 1) + ((u32)reloc->lobj.gpu_offset);
+                ib[idx+1] = radeon_get_ib_value(p, idx + 1) + ((u32)reloc->gpu_offset);
                 track->arrays[i + 0].robj = reloc->robj;
                 track->arrays[i + 0].esize = idx_value >> 8;
                 track->arrays[i + 0].esize &= 0x7F;
@@ -1595,7 +1594,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                 track->zb.robj = reloc->robj;
                 track->zb.offset = idx_value;
                 track->zb_dirty = true;
-                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 break;
         case RADEON_RB3D_COLOROFFSET:
                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
@@ -1608,7 +1607,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                 track->cb[0].robj = reloc->robj;
                 track->cb[0].offset = idx_value;
                 track->cb_dirty = true;
-                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 break;
         case RADEON_PP_TXOFFSET_0:
         case RADEON_PP_TXOFFSET_1:
@@ -1622,16 +1621,16 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                         return r;
                 }
                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO)
+                        if (reloc->tiling_flags & RADEON_TILING_MACRO)
                                 tile_flags |= RADEON_TXO_MACRO_TILE;
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO)
+                        if (reloc->tiling_flags & RADEON_TILING_MICRO)
                                 tile_flags |= RADEON_TXO_MICRO_TILE_X2;
 
                         tmp = idx_value & ~(0x7 << 2);
                         tmp |= tile_flags;
-                        ib[idx] = tmp + ((u32)reloc->lobj.gpu_offset);
+                        ib[idx] = tmp + ((u32)reloc->gpu_offset);
                 } else
-                        ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                        ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 track->textures[i].robj = reloc->robj;
                 track->tex_dirty = true;
                 break;
@@ -1649,7 +1648,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                         return r;
                 }
                 track->textures[0].cube_info[i].offset = idx_value;
-                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 track->textures[0].cube_info[i].robj = reloc->robj;
                 track->tex_dirty = true;
                 break;
@@ -1667,7 +1666,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                         return r;
                 }
                 track->textures[1].cube_info[i].offset = idx_value;
-                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 track->textures[1].cube_info[i].robj = reloc->robj;
                 track->tex_dirty = true;
                 break;
@@ -1685,7 +1684,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                         return r;
                 }
                 track->textures[2].cube_info[i].offset = idx_value;
-                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 track->textures[2].cube_info[i].robj = reloc->robj;
                 track->tex_dirty = true;
                 break;
@@ -1703,9 +1702,9 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                         return r;
                 }
                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO)
+                        if (reloc->tiling_flags & RADEON_TILING_MACRO)
                                 tile_flags |= RADEON_COLOR_TILE_ENABLE;
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO)
+                        if (reloc->tiling_flags & RADEON_TILING_MICRO)
                                 tile_flags |= RADEON_COLOR_MICROTILE_ENABLE;
 
                         tmp = idx_value & ~(0x7 << 16);
@@ -1773,7 +1772,7 @@ static int r100_packet0_check(struct radeon_cs_parser *p,
                         radeon_cs_dump_packet(p, pkt);
                         return r;
                 }
-                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 break;
         case RADEON_PP_CNTL:
                 {
@@ -1933,7 +1932,7 @@ static int r100_packet3_check(struct radeon_cs_parser *p,
                         radeon_cs_dump_packet(p, pkt);
                         return r;
                 }
-                ib[idx+1] = radeon_get_ib_value(p, idx+1) + ((u32)reloc->lobj.gpu_offset);
+                ib[idx+1] = radeon_get_ib_value(p, idx+1) + ((u32)reloc->gpu_offset);
                 r = r100_cs_track_check_pkt3_indx_buffer(p, pkt, reloc->robj);
                 if (r) {
                         return r;
@@ -1947,7 +1946,7 @@ static int r100_packet3_check(struct radeon_cs_parser *p,
                         radeon_cs_dump_packet(p, pkt);
                         return r;
                 }
-                ib[idx] = radeon_get_ib_value(p, idx) + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = radeon_get_ib_value(p, idx) + ((u32)reloc->gpu_offset);
                 track->num_arrays = 1;
                 track->vtx_size = r100_get_vtx_size(radeon_get_ib_value(p, idx + 2));
 
@@ -2523,11 +2522,9 @@ bool r100_gpu_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
 
         rbbm_status = RREG32(R_000E40_RBBM_STATUS);
         if (!G_000E40_GUI_ACTIVE(rbbm_status)) {
-                radeon_ring_lockup_update(ring);
+                radeon_ring_lockup_update(rdev, ring);
                 return false;
         }
-        /* force CP activities */
-        radeon_ring_force_activity(rdev, ring);
         return radeon_ring_test_lockup(rdev, ring);
 }
 
@@ -3223,12 +3220,12 @@ void r100_bandwidth_update(struct radeon_device *rdev)
 
         if (rdev->mode_info.crtcs[0]->base.enabled) {
                 mode1 = &rdev->mode_info.crtcs[0]->base.mode;
-                pixel_bytes1 = rdev->mode_info.crtcs[0]->base.fb->bits_per_pixel / 8;
+                pixel_bytes1 = rdev->mode_info.crtcs[0]->base.primary->fb->bits_per_pixel / 8;
         }
         if (!(rdev->flags & RADEON_SINGLE_CRTC)) {
                 if (rdev->mode_info.crtcs[1]->base.enabled) {
                         mode2 = &rdev->mode_info.crtcs[1]->base.mode;
-                        pixel_bytes2 = rdev->mode_info.crtcs[1]->base.fb->bits_per_pixel / 8;
+                        pixel_bytes2 = rdev->mode_info.crtcs[1]->base.primary->fb->bits_per_pixel / 8;
                 }
         }
 
diff --git a/drivers/gpu/drm/radeon/r200.c b/drivers/gpu/drm/radeon/r200.c
index b3807edb1936..58f0473aa73f 100644
--- a/drivers/gpu/drm/radeon/r200.c
+++ b/drivers/gpu/drm/radeon/r200.c
@@ -185,7 +185,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                 track->zb.robj = reloc->robj;
                 track->zb.offset = idx_value;
                 track->zb_dirty = true;
-                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 break;
         case RADEON_RB3D_COLOROFFSET:
                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
@@ -198,7 +198,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                 track->cb[0].robj = reloc->robj;
                 track->cb[0].offset = idx_value;
                 track->cb_dirty = true;
-                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 break;
         case R200_PP_TXOFFSET_0:
         case R200_PP_TXOFFSET_1:
@@ -215,16 +215,16 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                         return r;
                 }
                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO)
+                        if (reloc->tiling_flags & RADEON_TILING_MACRO)
                                 tile_flags |= R200_TXO_MACRO_TILE;
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO)
+                        if (reloc->tiling_flags & RADEON_TILING_MICRO)
                                 tile_flags |= R200_TXO_MICRO_TILE;
 
                         tmp = idx_value & ~(0x7 << 2);
                         tmp |= tile_flags;
-                        ib[idx] = tmp + ((u32)reloc->lobj.gpu_offset);
+                        ib[idx] = tmp + ((u32)reloc->gpu_offset);
                 } else
-                        ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                        ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 track->textures[i].robj = reloc->robj;
                 track->tex_dirty = true;
                 break;
@@ -268,7 +268,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                         return r;
                 }
                 track->textures[i].cube_info[face - 1].offset = idx_value;
-                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 track->textures[i].cube_info[face - 1].robj = reloc->robj;
                 track->tex_dirty = true;
                 break;
@@ -287,9 +287,9 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                 }
 
                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO)
+                        if (reloc->tiling_flags & RADEON_TILING_MACRO)
                                 tile_flags |= RADEON_COLOR_TILE_ENABLE;
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO)
+                        if (reloc->tiling_flags & RADEON_TILING_MICRO)
                                 tile_flags |= RADEON_COLOR_MICROTILE_ENABLE;
 
                         tmp = idx_value & ~(0x7 << 16);
@@ -362,7 +362,7 @@ int r200_packet0_check(struct radeon_cs_parser *p,
                         radeon_cs_dump_packet(p, pkt);
                         return r;
                 }
-                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 break;
         case RADEON_PP_CNTL:
                 {
diff --git a/drivers/gpu/drm/radeon/r300.c b/drivers/gpu/drm/radeon/r300.c
index 0b658b34b33a..206caf9700b7 100644
--- a/drivers/gpu/drm/radeon/r300.c
+++ b/drivers/gpu/drm/radeon/r300.c
@@ -640,7 +640,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                 track->cb[i].robj = reloc->robj;
                 track->cb[i].offset = idx_value;
                 track->cb_dirty = true;
-                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 break;
         case R300_ZB_DEPTHOFFSET:
                 r = radeon_cs_packet_next_reloc(p, &reloc, 0);
@@ -653,7 +653,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                 track->zb.robj = reloc->robj;
                 track->zb.offset = idx_value;
                 track->zb_dirty = true;
-                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 break;
         case R300_TX_OFFSET_0:
         case R300_TX_OFFSET_0+4:
@@ -682,16 +682,16 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
 
                 if (p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS) {
                         ib[idx] = (idx_value & 31) | /* keep the 1st 5 bits */
-                                  ((idx_value & ~31) + (u32)reloc->lobj.gpu_offset);
+                                  ((idx_value & ~31) + (u32)reloc->gpu_offset);
                 } else {
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO)
+                        if (reloc->tiling_flags & RADEON_TILING_MACRO)
                                 tile_flags |= R300_TXO_MACRO_TILE;
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO)
+                        if (reloc->tiling_flags & RADEON_TILING_MICRO)
                                 tile_flags |= R300_TXO_MICRO_TILE;
-                        else if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO_SQUARE)
+                        else if (reloc->tiling_flags & RADEON_TILING_MICRO_SQUARE)
                                 tile_flags |= R300_TXO_MICRO_TILE_SQUARE;
 
-                        tmp = idx_value + ((u32)reloc->lobj.gpu_offset);
+                        tmp = idx_value + ((u32)reloc->gpu_offset);
                         tmp |= tile_flags;
                         ib[idx] = tmp;
                 }
@@ -753,11 +753,11 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                                 return r;
                         }
 
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO)
+                        if (reloc->tiling_flags & RADEON_TILING_MACRO)
                                 tile_flags |= R300_COLOR_TILE_ENABLE;
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO)
+                        if (reloc->tiling_flags & RADEON_TILING_MICRO)
                                 tile_flags |= R300_COLOR_MICROTILE_ENABLE;
-                        else if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO_SQUARE)
+                        else if (reloc->tiling_flags & RADEON_TILING_MICRO_SQUARE)
                                 tile_flags |= R300_COLOR_MICROTILE_SQUARE_ENABLE;
 
                         tmp = idx_value & ~(0x7 << 16);
@@ -838,11 +838,11 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                                 return r;
                         }
 
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO)
+                        if (reloc->tiling_flags & RADEON_TILING_MACRO)
                                 tile_flags |= R300_DEPTHMACROTILE_ENABLE;
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO)
+                        if (reloc->tiling_flags & RADEON_TILING_MICRO)
                                 tile_flags |= R300_DEPTHMICROTILE_TILED;
-                        else if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO_SQUARE)
+                        else if (reloc->tiling_flags & RADEON_TILING_MICRO_SQUARE)
                                 tile_flags |= R300_DEPTHMICROTILE_TILED_SQUARE;
 
                         tmp = idx_value & ~(0x7 << 16);
@@ -1052,7 +1052,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                         radeon_cs_dump_packet(p, pkt);
                         return r;
                 }
-                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 break;
         case 0x4e0c:
                 /* RB3D_COLOR_CHANNEL_MASK */
@@ -1097,7 +1097,7 @@ static int r300_packet0_check(struct radeon_cs_parser *p,
                 track->aa.robj = reloc->robj;
                 track->aa.offset = idx_value;
                 track->aa_dirty = true;
-                ib[idx] = idx_value + ((u32)reloc->lobj.gpu_offset);
+                ib[idx] = idx_value + ((u32)reloc->gpu_offset);
                 break;
         case R300_RB3D_AARESOLVE_PITCH:
                 track->aa.pitch = idx_value & 0x3FFE;
@@ -1162,7 +1162,7 @@ static int r300_packet3_check(struct radeon_cs_parser *p,
                         radeon_cs_dump_packet(p, pkt);
                         return r;
                 }
-                ib[idx+1] = radeon_get_ib_value(p, idx + 1) + ((u32)reloc->lobj.gpu_offset);
+                ib[idx+1] = radeon_get_ib_value(p, idx + 1) + ((u32)reloc->gpu_offset);
                 r = r100_cs_track_check_pkt3_indx_buffer(p, pkt, reloc->robj);
                 if (r) {
                         return r;
diff --git a/drivers/gpu/drm/radeon/r600.c b/drivers/gpu/drm/radeon/r600.c
index 647ef4079217..6e887d004eba 100644
--- a/drivers/gpu/drm/radeon/r600.c
+++ b/drivers/gpu/drm/radeon/r600.c
@@ -1748,11 +1748,9 @@ bool r600_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
         if (!(reset_mask & (RADEON_RESET_GFX |
                             RADEON_RESET_COMPUTE |
                             RADEON_RESET_CP))) {
-                radeon_ring_lockup_update(ring);
+                radeon_ring_lockup_update(rdev, ring);
                 return false;
         }
-        /* force CP activities */
-        radeon_ring_force_activity(rdev, ring);
         return radeon_ring_test_lockup(rdev, ring);
 }
 
@@ -2604,8 +2602,6 @@ int r600_cp_resume(struct radeon_device *rdev)
         WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
         WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
 
-        ring->rptr = RREG32(CP_RB_RPTR);
-
         r600_cp_start(rdev);
         ring->ready = true;
         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
diff --git a/drivers/gpu/drm/radeon/r600_cs.c b/drivers/gpu/drm/radeon/r600_cs.c
index 2812c7d1ae6f..12511bb5fd6f 100644
--- a/drivers/gpu/drm/radeon/r600_cs.c
+++ b/drivers/gpu/drm/radeon/r600_cs.c
@@ -1022,7 +1022,7 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                                         "0x%04X\n", reg);
                         return -EINVAL;
                 }
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 break;
         case SQ_CONFIG:
                 track->sq_config = radeon_get_ib_value(p, idx);
@@ -1043,7 +1043,7 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         track->db_depth_info = radeon_get_ib_value(p, idx);
                         ib[idx] &= C_028010_ARRAY_MODE;
                         track->db_depth_info &= C_028010_ARRAY_MODE;
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) {
+                        if (reloc->tiling_flags & RADEON_TILING_MACRO) {
                                 ib[idx] |= S_028010_ARRAY_MODE(V_028010_ARRAY_2D_TILED_THIN1);
                                 track->db_depth_info |= S_028010_ARRAY_MODE(V_028010_ARRAY_2D_TILED_THIN1);
                         } else {
@@ -1084,9 +1084,9 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                 }
                 tmp = (reg - VGT_STRMOUT_BUFFER_BASE_0) / 16;
                 track->vgt_strmout_bo_offset[tmp] = radeon_get_ib_value(p, idx) << 8;
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 track->vgt_strmout_bo[tmp] = reloc->robj;
-                track->vgt_strmout_bo_mc[tmp] = reloc->lobj.gpu_offset;
+                track->vgt_strmout_bo_mc[tmp] = reloc->gpu_offset;
                 track->streamout_dirty = true;
                 break;
         case VGT_STRMOUT_BUFFER_SIZE_0:
@@ -1105,7 +1105,7 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                                         "0x%04X\n", reg);
                         return -EINVAL;
                 }
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 break;
         case R_028238_CB_TARGET_MASK:
                 track->cb_target_mask = radeon_get_ib_value(p, idx);
@@ -1142,10 +1142,10 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         }
                         tmp = (reg - R_0280A0_CB_COLOR0_INFO) / 4;
                         track->cb_color_info[tmp] = radeon_get_ib_value(p, idx);
-                        if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO) {
+                        if (reloc->tiling_flags & RADEON_TILING_MACRO) {
                                 ib[idx] |= S_0280A0_ARRAY_MODE(V_0280A0_ARRAY_2D_TILED_THIN1);
                                 track->cb_color_info[tmp] |= S_0280A0_ARRAY_MODE(V_0280A0_ARRAY_2D_TILED_THIN1);
-                        } else if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO) {
+                        } else if (reloc->tiling_flags & RADEON_TILING_MICRO) {
                                 ib[idx] |= S_0280A0_ARRAY_MODE(V_0280A0_ARRAY_1D_TILED_THIN1);
                                 track->cb_color_info[tmp] |= S_0280A0_ARRAY_MODE(V_0280A0_ARRAY_1D_TILED_THIN1);
                         }
@@ -1214,7 +1214,7 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         }
                         track->cb_color_frag_bo[tmp] = reloc->robj;
                         track->cb_color_frag_offset[tmp] = (u64)ib[idx] << 8;
-                        ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                        ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 }
                 if (G_0280A0_TILE_MODE(track->cb_color_info[tmp])) {
                         track->cb_dirty = true;
@@ -1245,7 +1245,7 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         }
                         track->cb_color_tile_bo[tmp] = reloc->robj;
                         track->cb_color_tile_offset[tmp] = (u64)ib[idx] << 8;
-                        ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                        ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 }
                 if (G_0280A0_TILE_MODE(track->cb_color_info[tmp])) {
                         track->cb_dirty = true;
@@ -1281,10 +1281,10 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                 }
                 tmp = (reg - CB_COLOR0_BASE) / 4;
                 track->cb_color_bo_offset[tmp] = radeon_get_ib_value(p, idx) << 8;
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 track->cb_color_base_last[tmp] = ib[idx];
                 track->cb_color_bo[tmp] = reloc->robj;
-                track->cb_color_bo_mc[tmp] = reloc->lobj.gpu_offset;
+                track->cb_color_bo_mc[tmp] = reloc->gpu_offset;
                 track->cb_dirty = true;
                 break;
         case DB_DEPTH_BASE:
@@ -1295,9 +1295,9 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         return -EINVAL;
                 }
                 track->db_offset = radeon_get_ib_value(p, idx) << 8;
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 track->db_bo = reloc->robj;
-                track->db_bo_mc = reloc->lobj.gpu_offset;
+                track->db_bo_mc = reloc->gpu_offset;
                 track->db_dirty = true;
                 break;
         case DB_HTILE_DATA_BASE:
@@ -1308,7 +1308,7 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                         return -EINVAL;
                 }
                 track->htile_offset = radeon_get_ib_value(p, idx) << 8;
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 track->htile_bo = reloc->robj;
                 track->db_dirty = true;
                 break;
@@ -1377,7 +1377,7 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                                         "0x%04X\n", reg);
                         return -EINVAL;
                 }
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 break;
         case SX_MEMORY_EXPORT_BASE:
                 r = radeon_cs_packet_next_reloc(p, &reloc, r600_nomm);
@@ -1386,7 +1386,7 @@ static int r600_cs_check_reg(struct radeon_cs_parser *p, u32 reg, u32 idx)
                                         "0x%04X\n", reg);
                         return -EINVAL;
                 }
-                ib[idx] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                ib[idx] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 break;
         case SX_MISC:
                 track->sx_misc_kill_all_prims = (radeon_get_ib_value(p, idx) & 0x1) != 0;
@@ -1672,7 +1672,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                         return -EINVAL;
                 }
 
-                offset = reloc->lobj.gpu_offset +
+                offset = reloc->gpu_offset +
                          (idx_value & 0xfffffff0) +
                          ((u64)(tmp & 0xff) << 32);
 
@@ -1713,7 +1713,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                         return -EINVAL;
                 }
 
-                offset = reloc->lobj.gpu_offset +
+                offset = reloc->gpu_offset +
                          idx_value +
                          ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32);
 
@@ -1765,7 +1765,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                                 return -EINVAL;
                         }
 
-                        offset = reloc->lobj.gpu_offset +
+                        offset = reloc->gpu_offset +
                                  (radeon_get_ib_value(p, idx+1) & 0xfffffff0) +
                                  ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
 
@@ -1805,7 +1805,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                         tmp = radeon_get_ib_value(p, idx) +
                                 ((u64)(radeon_get_ib_value(p, idx+1) & 0xff) << 32);
 
-                        offset = reloc->lobj.gpu_offset + tmp;
+                        offset = reloc->gpu_offset + tmp;
 
                         if ((tmp + size) > radeon_bo_size(reloc->robj)) {
                                 dev_warn(p->dev, "CP DMA src buffer too small (%llu %lu)\n",
@@ -1835,7 +1835,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                         tmp = radeon_get_ib_value(p, idx+2) +
                                 ((u64)(radeon_get_ib_value(p, idx+3) & 0xff) << 32);
 
-                        offset = reloc->lobj.gpu_offset + tmp;
+                        offset = reloc->gpu_offset + tmp;
 
                         if ((tmp + size) > radeon_bo_size(reloc->robj)) {
                                 dev_warn(p->dev, "CP DMA dst buffer too small (%llu %lu)\n",
@@ -1861,7 +1861,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                                 DRM_ERROR("bad SURFACE_SYNC\n");
                                 return -EINVAL;
                         }
-                        ib[idx+2] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                        ib[idx+2] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 }
                 break;
         case PACKET3_EVENT_WRITE:
@@ -1877,7 +1877,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                                 DRM_ERROR("bad EVENT_WRITE\n");
                                 return -EINVAL;
                         }
-                        offset = reloc->lobj.gpu_offset +
+                        offset = reloc->gpu_offset +
                                  (radeon_get_ib_value(p, idx+1) & 0xfffffff8) +
                                  ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
 
@@ -1899,7 +1899,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                         return -EINVAL;
                 }
 
-                offset = reloc->lobj.gpu_offset +
+                offset = reloc->gpu_offset +
                          (radeon_get_ib_value(p, idx+1) & 0xfffffffc) +
                          ((u64)(radeon_get_ib_value(p, idx+2) & 0xff) << 32);
 
@@ -1964,11 +1964,11 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                                         DRM_ERROR("bad SET_RESOURCE\n");
                                         return -EINVAL;
                                 }
-                                base_offset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                                base_offset = (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                                 if (!(p->cs_flags & RADEON_CS_KEEP_TILING_FLAGS)) {
-                                        if (reloc->lobj.tiling_flags & RADEON_TILING_MACRO)
+                                        if (reloc->tiling_flags & RADEON_TILING_MACRO)
                                                 ib[idx+1+(i*7)+0] |= S_038000_TILE_MODE(V_038000_ARRAY_2D_TILED_THIN1);
-                                        else if (reloc->lobj.tiling_flags & RADEON_TILING_MICRO)
+                                        else if (reloc->tiling_flags & RADEON_TILING_MICRO)
                                                 ib[idx+1+(i*7)+0] |= S_038000_TILE_MODE(V_038000_ARRAY_1D_TILED_THIN1);
                                 }
                                 texture = reloc->robj;
@@ -1978,13 +1978,13 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                                         DRM_ERROR("bad SET_RESOURCE\n");
                                         return -EINVAL;
                                 }
-                                mip_offset = (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                                mip_offset = (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                                 mipmap = reloc->robj;
                                 r = r600_check_texture_resource(p,  idx+(i*7)+1,
                                                                 texture, mipmap,
                                                                 base_offset + radeon_get_ib_value(p, idx+1+(i*7)+2),
                                                                 mip_offset + radeon_get_ib_value(p, idx+1+(i*7)+3),
-                                                                reloc->lobj.tiling_flags);
+                                                                reloc->tiling_flags);
                                 if (r)
                                         return r;
                                 ib[idx+1+(i*7)+2] += base_offset;
@@ -2008,7 +2008,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                                         ib[idx+1+(i*7)+1] = radeon_bo_size(reloc->robj) - offset;
                                 }
 
-                                offset64 = reloc->lobj.gpu_offset + offset;
+                                offset64 = reloc->gpu_offset + offset;
                                 ib[idx+1+(i*8)+0] = offset64;
                                 ib[idx+1+(i*8)+2] = (ib[idx+1+(i*8)+2] & 0xffffff00) |
                                                     (upper_32_bits(offset64) & 0xff);
@@ -2118,7 +2118,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                                           offset + 4, radeon_bo_size(reloc->robj));
                                 return -EINVAL;
                         }
-                        ib[idx+1] += (u32)((reloc->lobj.gpu_offset >> 8) & 0xffffffff);
+                        ib[idx+1] += (u32)((reloc->gpu_offset >> 8) & 0xffffffff);
                 }
                 break;
         case PACKET3_SURFACE_BASE_UPDATE:
@@ -2151,7 +2151,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                                           offset + 4, radeon_bo_size(reloc->robj));
                                 return -EINVAL;
                         }
-                        offset += reloc->lobj.gpu_offset;
+                        offset += reloc->gpu_offset;
                         ib[idx+1] = offset;
                         ib[idx+2] = upper_32_bits(offset) & 0xff;
                 }
@@ -2170,7 +2170,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                                           offset + 4, radeon_bo_size(reloc->robj));
                                 return -EINVAL;
                         }
-                        offset += reloc->lobj.gpu_offset;
+                        offset += reloc->gpu_offset;
                         ib[idx+3] = offset;
                         ib[idx+4] = upper_32_bits(offset) & 0xff;
                 }
@@ -2199,7 +2199,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                                   offset + 8, radeon_bo_size(reloc->robj));
                         return -EINVAL;
                 }
-                offset += reloc->lobj.gpu_offset;
+                offset += reloc->gpu_offset;
                 ib[idx+0] = offset;
                 ib[idx+1] = upper_32_bits(offset) & 0xff;
                 break;
@@ -2224,7 +2224,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                                           offset + 4, radeon_bo_size(reloc->robj));
                                 return -EINVAL;
                         }
-                        offset += reloc->lobj.gpu_offset;
+                        offset += reloc->gpu_offset;
                         ib[idx+1] = offset;
                         ib[idx+2] = upper_32_bits(offset) & 0xff;
                 } else {
@@ -2248,7 +2248,7 @@ static int r600_packet3_check(struct radeon_cs_parser *p,
                                           offset + 4, radeon_bo_size(reloc->robj));
                                 return -EINVAL;
                         }
-                        offset += reloc->lobj.gpu_offset;
+                        offset += reloc->gpu_offset;
                         ib[idx+3] = offset;
                         ib[idx+4] = upper_32_bits(offset) & 0xff;
                 } else {
@@ -2505,14 +2505,14 @@ int r600_dma_cs_parse(struct radeon_cs_parser *p)
                                 dst_offset = radeon_get_ib_value(p, idx+1);
                                 dst_offset <<= 8;
 
-                                ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
+                                ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8);
                                 p->idx += count + 5;
                         } else {
                                 dst_offset = radeon_get_ib_value(p, idx+1);
                                 dst_offset |= ((u64)(radeon_get_ib_value(p, idx+2) & 0xff)) << 32;
 
-                                ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
-                                ib[idx+2] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
+                                ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc);
+                                ib[idx+2] += upper_32_bits(dst_reloc->gpu_offset) & 0xff;
                                 p->idx += count + 3;
                         }
                         if ((dst_offset + (count * 4)) > radeon_bo_size(dst_reloc->robj)) {
@@ -2539,22 +2539,22 @@ int r600_dma_cs_parse(struct radeon_cs_parser *p)
                                         /* tiled src, linear dst */
                                         src_offset = radeon_get_ib_value(p, idx+1);
                                         src_offset <<= 8;
-                                        ib[idx+1] += (u32)(src_reloc->lobj.gpu_offset >> 8);
+                                        ib[idx+1] += (u32)(src_reloc->gpu_offset >> 8);
 
                                         dst_offset = radeon_get_ib_value(p, idx+5);
                                         dst_offset |= ((u64)(radeon_get_ib_value(p, idx+6) & 0xff)) << 32;
-                                        ib[idx+5] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
-                                        ib[idx+6] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
+                                        ib[idx+5] += (u32)(dst_reloc->gpu_offset & 0xfffffffc);
+                                        ib[idx+6] += upper_32_bits(dst_reloc->gpu_offset) & 0xff;
                                 } else {
                                         /* linear src, tiled dst */
                                         src_offset = radeon_get_ib_value(p, idx+5);
                                         src_offset |= ((u64)(radeon_get_ib_value(p, idx+6) & 0xff)) << 32;
-                                        ib[idx+5] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
-                                        ib[idx+6] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
+                                        ib[idx+5] += (u32)(src_reloc->gpu_offset & 0xfffffffc);
+                                        ib[idx+6] += upper_32_bits(src_reloc->gpu_offset) & 0xff;
 
                                         dst_offset = radeon_get_ib_value(p, idx+1);
                                         dst_offset <<= 8;
-                                        ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset >> 8);
+                                        ib[idx+1] += (u32)(dst_reloc->gpu_offset >> 8);
                                 }
                                 p->idx += 7;
                         } else {
@@ -2564,10 +2564,10 @@ int r600_dma_cs_parse(struct radeon_cs_parser *p)
                                         dst_offset = radeon_get_ib_value(p, idx+1);
                                         dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0xff)) << 32;
 
-                                        ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
-                                        ib[idx+2] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
-                                        ib[idx+3] += upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff;
-                                        ib[idx+4] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
+                                        ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc);
+                                        ib[idx+2] += (u32)(src_reloc->gpu_offset & 0xfffffffc);
+                                        ib[idx+3] += upper_32_bits(dst_reloc->gpu_offset) & 0xff;
+                                        ib[idx+4] += upper_32_bits(src_reloc->gpu_offset) & 0xff;
                                         p->idx += 5;
                                 } else {
                                         src_offset = radeon_get_ib_value(p, idx+2);
@@ -2575,10 +2575,10 @@ int r600_dma_cs_parse(struct radeon_cs_parser *p)
                                         dst_offset = radeon_get_ib_value(p, idx+1);
                                         dst_offset |= ((u64)(radeon_get_ib_value(p, idx+3) & 0xff0000)) << 16;
 
-                                        ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
-                                        ib[idx+2] += (u32)(src_reloc->lobj.gpu_offset & 0xfffffffc);
-                                        ib[idx+3] += upper_32_bits(src_reloc->lobj.gpu_offset) & 0xff;
-                                        ib[idx+3] += (upper_32_bits(dst_reloc->lobj.gpu_offset) & 0xff) << 16;
+                                        ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc);
+                                        ib[idx+2] += (u32)(src_reloc->gpu_offset & 0xfffffffc);
+                                        ib[idx+3] += upper_32_bits(src_reloc->gpu_offset) & 0xff;
+                                        ib[idx+3] += (upper_32_bits(dst_reloc->gpu_offset) & 0xff) << 16;
                                         p->idx += 4;
                                 }
                         }
@@ -2610,8 +2610,8 @@ int r600_dma_cs_parse(struct radeon_cs_parser *p)
                                          dst_offset + (count * 4), radeon_bo_size(dst_reloc->robj));
                                 return -EINVAL;
                         }
-                        ib[idx+1] += (u32)(dst_reloc->lobj.gpu_offset & 0xfffffffc);
-                        ib[idx+3] += (upper_32_bits(dst_reloc->lobj.gpu_offset) << 16) & 0x00ff0000;
+                        ib[idx+1] += (u32)(dst_reloc->gpu_offset & 0xfffffffc);
+                        ib[idx+3] += (upper_32_bits(dst_reloc->gpu_offset) << 16) & 0x00ff0000;
                         p->idx += 4;
                         break;
                 case DMA_PACKET_NOP:
diff --git a/drivers/gpu/drm/radeon/r600_dma.c b/drivers/gpu/drm/radeon/r600_dma.c
index b2d4c91e6272..53fcb28f5578 100644
--- a/drivers/gpu/drm/radeon/r600_dma.c
+++ b/drivers/gpu/drm/radeon/r600_dma.c
@@ -176,8 +176,6 @@ int r600_dma_resume(struct radeon_device *rdev)
         ring->wptr = 0;
         WREG32(DMA_RB_WPTR, ring->wptr << 2);
 
-        ring->rptr = RREG32(DMA_RB_RPTR) >> 2;
-
         WREG32(DMA_RB_CNTL, rb_cntl | DMA_RB_ENABLE);
 
         ring->ready = true;
@@ -221,11 +219,9 @@ bool r600_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
         u32 reset_mask = r600_gpu_check_soft_reset(rdev);
 
         if (!(reset_mask & RADEON_RESET_DMA)) {
-                radeon_ring_lockup_update(ring);
+                radeon_ring_lockup_update(rdev, ring);
                 return false;
         }
-        /* force ring activities */
-        radeon_ring_force_activity(rdev, ring);
         return radeon_ring_test_lockup(rdev, ring);
 }
 
diff --git a/drivers/gpu/drm/radeon/r600_dpm.c b/drivers/gpu/drm/radeon/r600_dpm.c
index e4cc9b314ce9..cbf7e3269f84 100644
--- a/drivers/gpu/drm/radeon/r600_dpm.c
+++ b/drivers/gpu/drm/radeon/r600_dpm.c
@@ -834,6 +834,26 @@ static int r600_parse_clk_voltage_dep_table(struct radeon_clock_voltage_dependen
         return 0;
 }
 
+int r600_get_platform_caps(struct radeon_device *rdev)
+{
+        struct radeon_mode_info *mode_info = &rdev->mode_info;
+        union power_info *power_info;
+        int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
+        u16 data_offset;
+        u8 frev, crev;
+
+        if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
+                                   &frev, &crev, &data_offset))
+                return -EINVAL;
+        power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
+
+        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
+        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
+        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
+
+        return 0;
+}
+
 /* sizeof(ATOM_PPLIB_EXTENDEDHEADER) */
 #define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V2 12
 #define SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V3 14
@@ -1043,7 +1063,15 @@ int r600_parse_extended_power_table(struct radeon_device *rdev)
                                 (mode_info->atom_context->bios + data_offset +
                                  le16_to_cpu(ext_hdr->usVCETableOffset) + 1 +
                                  1 + array->ucNumEntries * sizeof(VCEClockInfo));
+                        ATOM_PPLIB_VCE_State_Table *states =
+                                (ATOM_PPLIB_VCE_State_Table *)
+                                (mode_info->atom_context->bios + data_offset +
+                                 le16_to_cpu(ext_hdr->usVCETableOffset) + 1 +
+                                 1 + (array->ucNumEntries * sizeof (VCEClockInfo)) +
+                                 1 + (limits->numEntries * sizeof(ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record)));
                         ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record *entry;
+                        ATOM_PPLIB_VCE_State_Record *state_entry;
+                        VCEClockInfo *vce_clk;
                         u32 size = limits->numEntries *
                                 sizeof(struct radeon_vce_clock_voltage_dependency_entry);
                         rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries =
@@ -1055,8 +1083,9 @@ int r600_parse_extended_power_table(struct radeon_device *rdev)
                         rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.count =
                                 limits->numEntries;
                         entry = &limits->entries[0];
+                        state_entry = &states->entries[0];
                         for (i = 0; i < limits->numEntries; i++) {
-                                VCEClockInfo *vce_clk = (VCEClockInfo *)
+                                vce_clk = (VCEClockInfo *)
                                         ((u8 *)&array->entries[0] +
                                          (entry->ucVCEClockInfoIndex * sizeof(VCEClockInfo)));
                                 rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[i].evclk =
@@ -1068,6 +1097,23 @@ int r600_parse_extended_power_table(struct radeon_device *rdev)
                                 entry = (ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record *)
                                         ((u8 *)entry + sizeof(ATOM_PPLIB_VCE_Clock_Voltage_Limit_Record));
                         }
+                        for (i = 0; i < states->numEntries; i++) {
+                                if (i >= RADEON_MAX_VCE_LEVELS)
+                                        break;
+                                vce_clk = (VCEClockInfo *)
+                                        ((u8 *)&array->entries[0] +
+                                         (state_entry->ucVCEClockInfoIndex * sizeof(VCEClockInfo)));
+                                rdev->pm.dpm.vce_states[i].evclk =
+                                        le16_to_cpu(vce_clk->usEVClkLow) | (vce_clk->ucEVClkHigh << 16);
+                                rdev->pm.dpm.vce_states[i].ecclk =
+                                        le16_to_cpu(vce_clk->usECClkLow) | (vce_clk->ucECClkHigh << 16);
+                                rdev->pm.dpm.vce_states[i].clk_idx =
+                                        state_entry->ucClockInfoIndex & 0x3f;
+                                rdev->pm.dpm.vce_states[i].pstate =
+                                        (state_entry->ucClockInfoIndex & 0xc0) >> 6;
+                                state_entry = (ATOM_PPLIB_VCE_State_Record *)
+                                        ((u8 *)state_entry + sizeof(ATOM_PPLIB_VCE_State_Record));
+                        }
                 }
                 if ((le16_to_cpu(ext_hdr->usSize) >= SIZE_OF_ATOM_PPLIB_EXTENDEDHEADER_V3) &&
                         ext_hdr->usUVDTableOffset) {
diff --git a/drivers/gpu/drm/radeon/r600_dpm.h b/drivers/gpu/drm/radeon/r600_dpm.h
index 07eab2b04e81..46b9d2a03018 100644
--- a/drivers/gpu/drm/radeon/r600_dpm.h
+++ b/drivers/gpu/drm/radeon/r600_dpm.h
@@ -215,6 +215,8 @@ void r600_stop_dpm(struct radeon_device *rdev);
 
 bool r600_is_internal_thermal_sensor(enum radeon_int_thermal_type sensor);
 
+int r600_get_platform_caps(struct radeon_device *rdev);
+
 int r600_parse_extended_power_table(struct radeon_device *rdev);
 void r600_free_extended_power_table(struct radeon_device *rdev);
 
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index e887d027b6d0..f21db7a0b34d 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -113,19 +113,16 @@ extern int radeon_hard_reset;
 #define RADEONFB_CONN_LIMIT                     4
 #define RADEON_BIOS_NUM_SCRATCH                 8
 
-/* max number of rings */
-#define RADEON_NUM_RINGS                        6
-
 /* fence seq are set to this number when signaled */
 #define RADEON_FENCE_SIGNALED_SEQ               0LL
 
 /* internal ring indices */
 /* r1xx+ has gfx CP ring */
-#define RADEON_RING_TYPE_GFX_INDEX      0
+#define RADEON_RING_TYPE_GFX_INDEX              0
 
 /* cayman has 2 compute CP rings */
-#define CAYMAN_RING_TYPE_CP1_INDEX      1
-#define CAYMAN_RING_TYPE_CP2_INDEX      2
+#define CAYMAN_RING_TYPE_CP1_INDEX              1
+#define CAYMAN_RING_TYPE_CP2_INDEX              2
 
 /* R600+ has an async dma ring */
 #define R600_RING_TYPE_DMA_INDEX                3
@@ -133,7 +130,17 @@ extern int radeon_hard_reset;
 #define CAYMAN_RING_TYPE_DMA1_INDEX             4
 
 /* R600+ */
-#define R600_RING_TYPE_UVD_INDEX        5
+#define R600_RING_TYPE_UVD_INDEX                5
+
+/* TN+ */
+#define TN_RING_TYPE_VCE1_INDEX                 6
+#define TN_RING_TYPE_VCE2_INDEX                 7
+
+/* max number of rings */
+#define RADEON_NUM_RINGS                        8
+
+/* number of hw syncs before falling back on blocking */
+#define RADEON_NUM_SYNCS                        4
 
 /* number of hw syncs before falling back on blocking */
 #define RADEON_NUM_SYNCS                        4
@@ -356,9 +363,8 @@ int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence **fence, i
 void radeon_fence_process(struct radeon_device *rdev, int ring);
 bool radeon_fence_signaled(struct radeon_fence *fence);
 int radeon_fence_wait(struct radeon_fence *fence, bool interruptible);
-int radeon_fence_wait_locked(struct radeon_fence *fence);
-int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring);
-int radeon_fence_wait_empty_locked(struct radeon_device *rdev, int ring);
+int radeon_fence_wait_next(struct radeon_device *rdev, int ring);
+int radeon_fence_wait_empty(struct radeon_device *rdev, int ring);
 int radeon_fence_wait_any(struct radeon_device *rdev,
                           struct radeon_fence **fences,
                           bool intr);
@@ -450,6 +456,7 @@ struct radeon_bo {
         /* Protected by gem.mutex */
         struct list_head                list;
         /* Protected by tbo.reserved */
+        u32                             initial_domain;
         u32                             placements[3];
         struct ttm_placement            placement;
         struct ttm_buffer_object        tbo;
@@ -472,16 +479,6 @@ struct radeon_bo {
 };
 #define gem_to_radeon_bo(gobj) container_of((gobj), struct radeon_bo, gem_base)
 
-struct radeon_bo_list {
-        struct ttm_validate_buffer tv;
-        struct radeon_bo        *bo;
-        uint64_t                gpu_offset;
-        bool                    written;
-        unsigned                domain;
-        unsigned                alt_domain;
-        u32                     tiling_flags;
-};
-
 int radeon_gem_debugfs_init(struct radeon_device *rdev);
 
 /* sub-allocation manager, it has to be protected by another lock.
@@ -789,7 +786,6 @@ struct radeon_ib {
 struct radeon_ring {
         struct radeon_bo        *ring_obj;
         volatile uint32_t       *ring;
-        unsigned                rptr;
         unsigned                rptr_offs;
         unsigned                rptr_save_reg;
         u64                     next_rptr_gpu_addr;
@@ -799,8 +795,8 @@ struct radeon_ring {
         unsigned                ring_size;
         unsigned                ring_free_dw;
         int                     count_dw;
-        unsigned long           last_activity;
-        unsigned                last_rptr;
+        atomic_t                last_rptr;
+        atomic64_t              last_activity;
         uint64_t                gpu_addr;
         uint32_t                align_mask;
         uint32_t                ptr_mask;
@@ -852,17 +848,22 @@ struct radeon_mec {
 #define R600_PTE_READABLE       (1 << 5)
 #define R600_PTE_WRITEABLE      (1 << 6)
 
+struct radeon_vm_pt {
+        struct radeon_bo                *bo;
+        uint64_t                        addr;
+};
+
 struct radeon_vm {
-        struct list_head                list;
         struct list_head                va;
         unsigned                        id;
 
         /* contains the page directory */
-        struct radeon_sa_bo             *page_directory;
+        struct radeon_bo                *page_directory;
         uint64_t                        pd_gpu_addr;
+        unsigned                        max_pde_used;
 
         /* array of page tables, one for each page directory entry */
-        struct radeon_sa_bo             **page_tables;
+        struct radeon_vm_pt             *page_tables;
 
         struct mutex                    mutex;
         /* last fence for cs using this vm */
@@ -874,10 +875,7 @@ struct radeon_vm {
 };
 
 struct radeon_vm_manager {
-        struct mutex                    lock;
-        struct list_head                lru_vm;
         struct radeon_fence             *active[RADEON_NUM_VM];
-        struct radeon_sa_manager        sa_manager;
         uint32_t                        max_pfn;
         /* number of VMIDs */
         unsigned                        nvm;
@@ -953,8 +951,8 @@ void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *c
 void radeon_ring_undo(struct radeon_ring *ring);
 void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *cp);
 int radeon_ring_test(struct radeon_device *rdev, struct radeon_ring *cp);
-void radeon_ring_force_activity(struct radeon_device *rdev, struct radeon_ring *ring);
-void radeon_ring_lockup_update(struct radeon_ring *ring);
+void radeon_ring_lockup_update(struct radeon_device *rdev,
+                               struct radeon_ring *ring);
 bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring);
 unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring,
                             uint32_t **data);
@@ -980,9 +978,12 @@ void cayman_dma_fini(struct radeon_device *rdev);
 struct radeon_cs_reloc {
         struct drm_gem_object           *gobj;
         struct radeon_bo                *robj;
-        struct radeon_bo_list           lobj;
+        struct ttm_validate_buffer      tv;
+        uint64_t                        gpu_offset;
+        unsigned                        domain;
+        unsigned                        alt_domain;
+        uint32_t                        tiling_flags;
         uint32_t                        handle;
-        uint32_t                        flags;
 };
 
 struct radeon_cs_chunk {
@@ -1006,6 +1007,7 @@ struct radeon_cs_parser {
         unsigned                nrelocs;
         struct radeon_cs_reloc  *relocs;
         struct radeon_cs_reloc  **relocs_ptr;
+        struct radeon_cs_reloc  *vm_bos;
         struct list_head        validated;
         unsigned                dma_reloc_idx;
         /* indices of various chunks */
@@ -1255,6 +1257,17 @@ enum radeon_dpm_event_src {
         RADEON_DPM_EVENT_SRC_DIGIAL_OR_EXTERNAL = 4
 };
 
+#define RADEON_MAX_VCE_LEVELS 6
+
+enum radeon_vce_level {
+        RADEON_VCE_LEVEL_AC_ALL = 0,     /* AC, All cases */
+        RADEON_VCE_LEVEL_DC_EE = 1,      /* DC, entropy encoding */
+        RADEON_VCE_LEVEL_DC_LL_LOW = 2,  /* DC, low latency queue, res <= 720 */
+        RADEON_VCE_LEVEL_DC_LL_HIGH = 3, /* DC, low latency queue, 1080 >= res > 720 */
+        RADEON_VCE_LEVEL_DC_GP_LOW = 4,  /* DC, general purpose queue, res <= 720 */
+        RADEON_VCE_LEVEL_DC_GP_HIGH = 5, /* DC, general purpose queue, 1080 >= res > 720 */
+};
+
 struct radeon_ps {
         u32 caps; /* vbios flags */
         u32 class; /* vbios flags */
@@ -1265,6 +1278,8 @@ struct radeon_ps {
         /* VCE clocks */
         u32 evclk;
         u32 ecclk;
+        bool vce_active;
+        enum radeon_vce_level vce_level;
         /* asic priv */
         void *ps_priv;
 };
@@ -1439,6 +1454,17 @@ enum radeon_dpm_forced_level {
         RADEON_DPM_FORCED_LEVEL_HIGH = 2,
 };
 
+struct radeon_vce_state {
+        /* vce clocks */
+        u32 evclk;
+        u32 ecclk;
+        /* gpu clocks */
+        u32 sclk;
+        u32 mclk;
+        u8 clk_idx;
+        u8 pstate;
+};
+
 struct radeon_dpm {
         struct radeon_ps        *ps;
         /* number of valid power states */
@@ -1451,6 +1477,9 @@ struct radeon_dpm {
         struct radeon_ps        *boot_ps;
         /* default uvd power state */
         struct radeon_ps        *uvd_ps;
+        /* vce requirements */
+        struct radeon_vce_state vce_states[RADEON_MAX_VCE_LEVELS];
+        enum radeon_vce_level vce_level;
         enum radeon_pm_state_type state;
         enum radeon_pm_state_type user_state;
         u32                     platform_caps;
@@ -1476,6 +1505,7 @@ struct radeon_dpm {
         /* special states active */
         bool                    thermal_active;
         bool                    uvd_active;
+        bool                    vce_active;
         /* thermal handling */
         struct radeon_dpm_thermal thermal;
         /* forced levels */
@@ -1486,6 +1516,7 @@ struct radeon_dpm {
 };
 
 void radeon_dpm_enable_uvd(struct radeon_device *rdev, bool enable);
+void radeon_dpm_enable_vce(struct radeon_device *rdev, bool enable);
 
 struct radeon_pm {
         struct mutex            mutex;
@@ -1591,6 +1622,45 @@ int radeon_uvd_calc_upll_dividers(struct radeon_device *rdev,
 int radeon_uvd_send_upll_ctlreq(struct radeon_device *rdev,
                                 unsigned cg_upll_func_cntl);
 
+/*
+ * VCE
+ */
+#define RADEON_MAX_VCE_HANDLES  16
+#define RADEON_VCE_STACK_SIZE   (1024*1024)
+#define RADEON_VCE_HEAP_SIZE    (4*1024*1024)
+
+struct radeon_vce {
+        struct radeon_bo        *vcpu_bo;
+        uint64_t                gpu_addr;
+        unsigned                fw_version;
+        unsigned                fb_version;
+        atomic_t                handles[RADEON_MAX_VCE_HANDLES];
+        struct drm_file         *filp[RADEON_MAX_VCE_HANDLES];
+        struct delayed_work     idle_work;
+};
+
+int radeon_vce_init(struct radeon_device *rdev);
+void radeon_vce_fini(struct radeon_device *rdev);
+int radeon_vce_suspend(struct radeon_device *rdev);
+int radeon_vce_resume(struct radeon_device *rdev);
+int radeon_vce_get_create_msg(struct radeon_device *rdev, int ring,
+                              uint32_t handle, struct radeon_fence **fence);
+int radeon_vce_get_destroy_msg(struct radeon_device *rdev, int ring,
+                               uint32_t handle, struct radeon_fence **fence);
+void radeon_vce_free_handles(struct radeon_device *rdev, struct drm_file *filp);
+void radeon_vce_note_usage(struct radeon_device *rdev);
+int radeon_vce_cs_reloc(struct radeon_cs_parser *p, int lo, int hi);
+int radeon_vce_cs_parse(struct radeon_cs_parser *p);
+bool radeon_vce_semaphore_emit(struct radeon_device *rdev,
+                               struct radeon_ring *ring,
+                               struct radeon_semaphore *semaphore,
+                               bool emit_wait);
+void radeon_vce_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib);
+void radeon_vce_fence_emit(struct radeon_device *rdev,
+                           struct radeon_fence *fence);
+int radeon_vce_ring_test(struct radeon_device *rdev, struct radeon_ring *ring);
+int radeon_vce_ib_test(struct radeon_device *rdev, struct radeon_ring *ring);
+
 struct r600_audio_pin {
         int                     channels;
         int                     rate;
@@ -1780,6 +1850,7 @@ struct radeon_asic {
                 void (*set_pcie_lanes)(struct radeon_device *rdev, int lanes);
                 void (*set_clock_gating)(struct radeon_device *rdev, int enable);
                 int (*set_uvd_clocks)(struct radeon_device *rdev, u32 vclk, u32 dclk);
+                int (*set_vce_clocks)(struct radeon_device *rdev, u32 evclk, u32 ecclk);
                 int (*get_temperature)(struct radeon_device *rdev);
         } pm;
         /* dynamic power management */
@@ -2041,6 +2112,8 @@ int radeon_gem_wait_idle_ioctl(struct drm_device *dev, void *data,
                               struct drm_file *filp);
 int radeon_gem_va_ioctl(struct drm_device *dev, void *data,
                           struct drm_file *filp);
+int radeon_gem_op_ioctl(struct drm_device *dev, void *data,
+                        struct drm_file *filp);
 int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp);
 int radeon_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
                                 struct drm_file *filp);
@@ -2186,6 +2259,7 @@ struct radeon_device {
         struct radeon_gem               gem;
         struct radeon_pm                pm;
         struct radeon_uvd               uvd;
+        struct radeon_vce               vce;
         uint32_t                        bios_scratch[RADEON_BIOS_NUM_SCRATCH];
         struct radeon_wb                wb;
         struct radeon_dummy_page        dummy_page;
@@ -2205,6 +2279,7 @@ struct radeon_device {
         const struct firmware *sdma_fw; /* CIK SDMA firmware */
         const struct firmware *smc_fw;  /* SMC firmware */
         const struct firmware *uvd_fw;  /* UVD firmware */
+        const struct firmware *vce_fw;  /* VCE firmware */
         struct r600_vram_scratch vram_scratch;
         int msi_enabled; /* msi enabled */
         struct r600_ih ih; /* r6/700 interrupt ring */
@@ -2229,6 +2304,10 @@ struct radeon_device {
         /* virtual memory */
         struct radeon_vm_manager        vm_manager;
         struct mutex                    gpu_clock_mutex;
+        /* memory stats */
+        atomic64_t                      vram_usage;
+        atomic64_t                      gtt_usage;
+        atomic64_t                      num_bytes_moved;
         /* ACPI interface */
         struct radeon_atif              atif;
         struct radeon_atcs              atcs;
@@ -2639,6 +2718,7 @@ void radeon_ring_write(struct radeon_ring *ring, uint32_t v);
 #define radeon_set_pcie_lanes(rdev, l) (rdev)->asic->pm.set_pcie_lanes((rdev), (l))
 #define radeon_set_clock_gating(rdev, e) (rdev)->asic->pm.set_clock_gating((rdev), (e))
 #define radeon_set_uvd_clocks(rdev, v, d) (rdev)->asic->pm.set_uvd_clocks((rdev), (v), (d))
+#define radeon_set_vce_clocks(rdev, ev, ec) (rdev)->asic->pm.set_vce_clocks((rdev), (ev), (ec))
 #define radeon_get_temperature(rdev) (rdev)->asic->pm.get_temperature((rdev))
 #define radeon_set_surface_reg(rdev, r, f, p, o, s) ((rdev)->asic->surface.set_reg((rdev), (r), (f), (p), (o), (s)))
 #define radeon_clear_surface_reg(rdev, r) ((rdev)->asic->surface.clear_reg((rdev), (r)))
@@ -2715,16 +2795,22 @@ extern void radeon_program_register_sequence(struct radeon_device *rdev,
  */
 int radeon_vm_manager_init(struct radeon_device *rdev);
 void radeon_vm_manager_fini(struct radeon_device *rdev);
-void radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm);
+int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm);
 void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm);
-int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm);
-void radeon_vm_add_to_lru(struct radeon_device *rdev, struct radeon_vm *vm);
+struct radeon_cs_reloc *radeon_vm_get_bos(struct radeon_device *rdev,
+                                          struct radeon_vm *vm,
+                                          struct list_head *head);
 struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev,
                                        struct radeon_vm *vm, int ring);
+void radeon_vm_flush(struct radeon_device *rdev,
+                     struct radeon_vm *vm,
+                     int ring);
 void radeon_vm_fence(struct radeon_device *rdev,
                      struct radeon_vm *vm,
                      struct radeon_fence *fence);
 uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr);
+int radeon_vm_update_page_directory(struct radeon_device *rdev,
+                                    struct radeon_vm *vm);
 int radeon_vm_bo_update(struct radeon_device *rdev,
                         struct radeon_vm *vm,
                         struct radeon_bo *bo,
diff --git a/drivers/gpu/drm/radeon/radeon_asic.c b/drivers/gpu/drm/radeon/radeon_asic.c
index dda02bfc10a4..b8a24a75d4ff 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.c
+++ b/drivers/gpu/drm/radeon/radeon_asic.c
@@ -1987,6 +1987,19 @@ static struct radeon_asic_ring ci_dma_ring = {
         .set_wptr = &cik_sdma_set_wptr,
 };
 
+static struct radeon_asic_ring ci_vce_ring = {
+        .ib_execute = &radeon_vce_ib_execute,
+        .emit_fence = &radeon_vce_fence_emit,
+        .emit_semaphore = &radeon_vce_semaphore_emit,
+        .cs_parse = &radeon_vce_cs_parse,
+        .ring_test = &radeon_vce_ring_test,
+        .ib_test = &radeon_vce_ib_test,
+        .is_lockup = &radeon_ring_test_lockup,
+        .get_rptr = &vce_v1_0_get_rptr,
+        .get_wptr = &vce_v1_0_get_wptr,
+        .set_wptr = &vce_v1_0_set_wptr,
+};
+
 static struct radeon_asic ci_asic = {
         .init = &cik_init,
         .fini = &cik_fini,
@@ -2015,6 +2028,8 @@ static struct radeon_asic ci_asic = {
                 [R600_RING_TYPE_DMA_INDEX] = &ci_dma_ring,
                 [CAYMAN_RING_TYPE_DMA1_INDEX] = &ci_dma_ring,
                 [R600_RING_TYPE_UVD_INDEX] = &cayman_uvd_ring,
+                [TN_RING_TYPE_VCE1_INDEX] = &ci_vce_ring,
+                [TN_RING_TYPE_VCE2_INDEX] = &ci_vce_ring,
         },
         .irq = {
                 .set = &cik_irq_set,
@@ -2061,6 +2076,7 @@ static struct radeon_asic ci_asic = {
                 .set_pcie_lanes = NULL,
                 .set_clock_gating = NULL,
                 .set_uvd_clocks = &cik_set_uvd_clocks,
+                .set_vce_clocks = &cik_set_vce_clocks,
                 .get_temperature = &ci_get_temp,
         },
         .dpm = {
@@ -2117,6 +2133,8 @@ static struct radeon_asic kv_asic = {
                 [R600_RING_TYPE_DMA_INDEX] = &ci_dma_ring,
                 [CAYMAN_RING_TYPE_DMA1_INDEX] = &ci_dma_ring,
                 [R600_RING_TYPE_UVD_INDEX] = &cayman_uvd_ring,
+                [TN_RING_TYPE_VCE1_INDEX] = &ci_vce_ring,
+                [TN_RING_TYPE_VCE2_INDEX] = &ci_vce_ring,
         },
         .irq = {
                 .set = &cik_irq_set,
@@ -2163,6 +2181,7 @@ static struct radeon_asic kv_asic = {
                 .set_pcie_lanes = NULL,
                 .set_clock_gating = NULL,
                 .set_uvd_clocks = &cik_set_uvd_clocks,
+                .set_vce_clocks = &cik_set_vce_clocks,
                 .get_temperature = &kv_get_temp,
         },
         .dpm = {
diff --git a/drivers/gpu/drm/radeon/radeon_asic.h b/drivers/gpu/drm/radeon/radeon_asic.h
index ae637cfda783..3d55a3a39e82 100644
--- a/drivers/gpu/drm/radeon/radeon_asic.h
+++ b/drivers/gpu/drm/radeon/radeon_asic.h
@@ -717,6 +717,7 @@ u32 cik_get_xclk(struct radeon_device *rdev);
 uint32_t cik_pciep_rreg(struct radeon_device *rdev, uint32_t reg);
 void cik_pciep_wreg(struct radeon_device *rdev, uint32_t reg, uint32_t v);
 int cik_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk);
+int cik_set_vce_clocks(struct radeon_device *rdev, u32 evclk, u32 ecclk);
 void cik_sdma_fence_ring_emit(struct radeon_device *rdev,
                               struct radeon_fence *fence);
 bool cik_sdma_semaphore_ring_emit(struct radeon_device *rdev,
@@ -863,4 +864,17 @@ bool uvd_v3_1_semaphore_emit(struct radeon_device *rdev,
 /* uvd v4.2 */
 int uvd_v4_2_resume(struct radeon_device *rdev);
 
+/* vce v1.0 */
+uint32_t vce_v1_0_get_rptr(struct radeon_device *rdev,
+                           struct radeon_ring *ring);
+uint32_t vce_v1_0_get_wptr(struct radeon_device *rdev,
+                           struct radeon_ring *ring);
+void vce_v1_0_set_wptr(struct radeon_device *rdev,
+                       struct radeon_ring *ring);
+int vce_v1_0_init(struct radeon_device *rdev);
+int vce_v1_0_start(struct radeon_device *rdev);
+
+/* vce v2.0 */
+int vce_v2_0_resume(struct radeon_device *rdev);
+
 #endif
diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
index 82d4f865546e..c566b486ca08 100644
--- a/drivers/gpu/drm/radeon/radeon_connectors.c
+++ b/drivers/gpu/drm/radeon/radeon_connectors.c
@@ -89,7 +89,7 @@ static void radeon_property_change_mode(struct drm_encoder *encoder)
 
         if (crtc && crtc->enabled) {
                 drm_crtc_helper_set_mode(crtc, &crtc->mode,
-                                         crtc->x, crtc->y, crtc->fb);
+                                         crtc->x, crtc->y, crtc->primary->fb);
         }
 }
 
@@ -1595,6 +1595,7 @@ radeon_add_atom_connector(struct drm_device *dev,
         uint32_t subpixel_order = SubPixelNone;
         bool shared_ddc = false;
         bool is_dp_bridge = false;
+        bool has_aux = false;
 
         if (connector_type == DRM_MODE_CONNECTOR_Unknown)
                 return;
@@ -1672,7 +1673,9 @@ radeon_add_atom_connector(struct drm_device *dev,
                                 radeon_dig_connector->dp_i2c_bus = radeon_i2c_create_dp(dev, i2c_bus, "eDP-auxch");
                         else
                                 radeon_dig_connector->dp_i2c_bus = radeon_i2c_create_dp(dev, i2c_bus, "DP-auxch");
-                        if (!radeon_dig_connector->dp_i2c_bus)
+                        if (radeon_dig_connector->dp_i2c_bus)
+                                has_aux = true;
+                        else
                                 DRM_ERROR("DP: Failed to assign dp ddc bus! Check dmesg for i2c errors.\n");
                         radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
                         if (!radeon_connector->ddc_bus)
@@ -1895,7 +1898,9 @@ radeon_add_atom_connector(struct drm_device *dev,
                                 if (!radeon_dig_connector->dp_i2c_bus)
                                         DRM_ERROR("DP: Failed to assign dp ddc bus! Check dmesg for i2c errors.\n");
                                 radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
-                                if (!radeon_connector->ddc_bus)
+                                if (radeon_connector->ddc_bus)
+                                        has_aux = true;
+                                else
                                         DRM_ERROR("DP: Failed to assign ddc bus! Check dmesg for i2c errors.\n");
                         }
                         subpixel_order = SubPixelHorizontalRGB;
@@ -1939,7 +1944,9 @@ radeon_add_atom_connector(struct drm_device *dev,
                         if (i2c_bus->valid) {
                                 /* add DP i2c bus */
                                 radeon_dig_connector->dp_i2c_bus = radeon_i2c_create_dp(dev, i2c_bus, "eDP-auxch");
-                                if (!radeon_dig_connector->dp_i2c_bus)
+                                if (radeon_dig_connector->dp_i2c_bus)
+                                        has_aux = true;
+                                else
                                         DRM_ERROR("DP: Failed to assign dp ddc bus! Check dmesg for i2c errors.\n");
                                 radeon_connector->ddc_bus = radeon_i2c_lookup(rdev, i2c_bus);
                                 if (!radeon_connector->ddc_bus)
@@ -2000,6 +2007,10 @@ radeon_add_atom_connector(struct drm_device *dev,
 
         connector->display_info.subpixel_order = subpixel_order;
         drm_sysfs_connector_add(connector);
+
+        if (has_aux)
+                radeon_dp_aux_init(radeon_connector);
+
         return;
 
 failed:
diff --git a/drivers/gpu/drm/radeon/radeon_cs.c b/drivers/gpu/drm/radeon/radeon_cs.c
index dfb5a1db87d4..2b6e0ebcc13a 100644
--- a/drivers/gpu/drm/radeon/radeon_cs.c
+++ b/drivers/gpu/drm/radeon/radeon_cs.c
@@ -24,16 +24,59 @@
  * Authors:
  *    Jerome Glisse <glisse@freedesktop.org>
  */
+#include <linux/list_sort.h>
 #include <drm/drmP.h>
 #include <drm/radeon_drm.h>
 #include "radeon_reg.h"
 #include "radeon.h"
 #include "radeon_trace.h"
 
+#define RADEON_CS_MAX_PRIORITY          32u
+#define RADEON_CS_NUM_BUCKETS           (RADEON_CS_MAX_PRIORITY + 1)
+
+/* This is based on the bucket sort with O(n) time complexity.
+ * An item with priority "i" is added to bucket[i]. The lists are then
+ * concatenated in descending order.
+ */
+struct radeon_cs_buckets {
+        struct list_head bucket[RADEON_CS_NUM_BUCKETS];
+};
+
+static void radeon_cs_buckets_init(struct radeon_cs_buckets *b)
+{
+        unsigned i;
+
+        for (i = 0; i < RADEON_CS_NUM_BUCKETS; i++)
+                INIT_LIST_HEAD(&b->bucket[i]);
+}
+
+static void radeon_cs_buckets_add(struct radeon_cs_buckets *b,
+                                  struct list_head *item, unsigned priority)
+{
+        /* Since buffers which appear sooner in the relocation list are
+         * likely to be used more often than buffers which appear later
+         * in the list, the sort mustn't change the ordering of buffers
+         * with the same priority, i.e. it must be stable.
+         */
+        list_add_tail(item, &b->bucket[min(priority, RADEON_CS_MAX_PRIORITY)]);
+}
+
+static void radeon_cs_buckets_get_list(struct radeon_cs_buckets *b,
+                                       struct list_head *out_list)
+{
+        unsigned i;
+
+        /* Connect the sorted buckets in the output list. */
+        for (i = 0; i < RADEON_CS_NUM_BUCKETS; i++) {
+                list_splice(&b->bucket[i], out_list);
+        }
+}
+
 static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
 {
         struct drm_device *ddev = p->rdev->ddev;
         struct radeon_cs_chunk *chunk;
+        struct radeon_cs_buckets buckets;
         unsigned i, j;
         bool duplicate;
 
@@ -52,8 +95,12 @@ static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
         if (p->relocs == NULL) {
                 return -ENOMEM;
         }
+
+        radeon_cs_buckets_init(&buckets);
+
         for (i = 0; i < p->nrelocs; i++) {
                 struct drm_radeon_cs_reloc *r;
+                unsigned priority;
 
                 duplicate = false;
                 r = (struct drm_radeon_cs_reloc *)&chunk->kdata[i*4];
@@ -78,8 +125,14 @@ static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
                 }
                 p->relocs_ptr[i] = &p->relocs[i];
                 p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj);
-                p->relocs[i].lobj.bo = p->relocs[i].robj;
-                p->relocs[i].lobj.written = !!r->write_domain;
+
+                /* The userspace buffer priorities are from 0 to 15. A higher
+                 * number means the buffer is more important.
+                 * Also, the buffers used for write have a higher priority than
+                 * the buffers used for read only, which doubles the range
+                 * to 0 to 31. 32 is reserved for the kernel driver.
+                 */
+                priority = (r->flags & 0xf) * 2 + !!r->write_domain;
 
                 /* the first reloc of an UVD job is the msg and that must be in
                    VRAM, also but everything into VRAM on AGP cards to avoid
@@ -87,29 +140,38 @@ static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
                 if (p->ring == R600_RING_TYPE_UVD_INDEX &&
                     (i == 0 || drm_pci_device_is_agp(p->rdev->ddev))) {
                         /* TODO: is this still needed for NI+ ? */
-                        p->relocs[i].lobj.domain =
+                        p->relocs[i].domain =
                                 RADEON_GEM_DOMAIN_VRAM;
 
-                        p->relocs[i].lobj.alt_domain =
+                        p->relocs[i].alt_domain =
                                 RADEON_GEM_DOMAIN_VRAM;
 
+                        /* prioritize this over any other relocation */
+                        priority = RADEON_CS_MAX_PRIORITY;
                 } else {
                         uint32_t domain = r->write_domain ?
                                 r->write_domain : r->read_domains;
 
-                        p->relocs[i].lobj.domain = domain;
+                        p->relocs[i].domain = domain;
                         if (domain == RADEON_GEM_DOMAIN_VRAM)
                                 domain |= RADEON_GEM_DOMAIN_GTT;
-                        p->relocs[i].lobj.alt_domain = domain;
+                        p->relocs[i].alt_domain = domain;
                 }
 
-                p->relocs[i].lobj.tv.bo = &p->relocs[i].robj->tbo;
+                p->relocs[i].tv.bo = &p->relocs[i].robj->tbo;
                 p->relocs[i].handle = r->handle;
 
-                radeon_bo_list_add_object(&p->relocs[i].lobj,
-                                          &p->validated);
+                radeon_cs_buckets_add(&buckets, &p->relocs[i].tv.head,
+                                      priority);
         }
-        return radeon_bo_list_validate(&p->ticket, &p->validated, p->ring);
+
+        radeon_cs_buckets_get_list(&buckets, &p->validated);
+
+        if (p->cs_flags & RADEON_CS_USE_VM)
+                p->vm_bos = radeon_vm_get_bos(p->rdev, p->ib.vm,
+                                              &p->validated);
+
+        return radeon_bo_list_validate(p->rdev, &p->ticket, &p->validated, p->ring);
 }
 
 static int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority)
@@ -147,6 +209,10 @@ static int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority
         case RADEON_CS_RING_UVD:
                 p->ring = R600_RING_TYPE_UVD_INDEX;
                 break;
+        case RADEON_CS_RING_VCE:
+                /* TODO: only use the low priority ring for now */
+                p->ring = TN_RING_TYPE_VCE1_INDEX;
+                break;
         }
         return 0;
 }
@@ -286,6 +352,16 @@ int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)
         return 0;
 }
 
+static int cmp_size_smaller_first(void *priv, struct list_head *a,
+                                  struct list_head *b)
+{
+        struct radeon_cs_reloc *la = list_entry(a, struct radeon_cs_reloc, tv.head);
+        struct radeon_cs_reloc *lb = list_entry(b, struct radeon_cs_reloc, tv.head);
+
+        /* Sort A before B if A is smaller. */
+        return (int)la->robj->tbo.num_pages - (int)lb->robj->tbo.num_pages;
+}
+
 /**
  * cs_parser_fini() - clean parser states
  * @parser:     parser structure holding parsing context.
@@ -299,6 +375,18 @@ static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error, bo
         unsigned i;
 
         if (!error) {
+                /* Sort the buffer list from the smallest to largest buffer,
+                 * which affects the order of buffers in the LRU list.
+                 * This assures that the smallest buffers are added first
+                 * to the LRU list, so they are likely to be later evicted
+                 * first, instead of large buffers whose eviction is more
+                 * expensive.
+                 *
+                 * This slightly lowers the number of bytes moved by TTM
+                 * per frame under memory pressure.
+                 */
+                list_sort(NULL, &parser->validated, cmp_size_smaller_first);
+
                 ttm_eu_fence_buffer_objects(&parser->ticket,
                                             &parser->validated,
                                             parser->ib.fence);
@@ -316,6 +404,7 @@ static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error, bo
         kfree(parser->track);
         kfree(parser->relocs);
         kfree(parser->relocs_ptr);
+        kfree(parser->vm_bos);
         for (i = 0; i < parser->nchunks; i++)
                 drm_free_large(parser->chunks[i].kdata);
         kfree(parser->chunks);
@@ -343,6 +432,9 @@ static int radeon_cs_ib_chunk(struct radeon_device *rdev,
 
         if (parser->ring == R600_RING_TYPE_UVD_INDEX)
                 radeon_uvd_note_usage(rdev);
+        else if ((parser->ring == TN_RING_TYPE_VCE1_INDEX) ||
+                 (parser->ring == TN_RING_TYPE_VCE2_INDEX))
+                radeon_vce_note_usage(rdev);
 
         radeon_cs_sync_rings(parser);
         r = radeon_ib_schedule(rdev, &parser->ib, NULL);
@@ -352,24 +444,32 @@ static int radeon_cs_ib_chunk(struct radeon_device *rdev,
         return r;
 }
 
-static int radeon_bo_vm_update_pte(struct radeon_cs_parser *parser,
+static int radeon_bo_vm_update_pte(struct radeon_cs_parser *p,
                                    struct radeon_vm *vm)
 {
-        struct radeon_device *rdev = parser->rdev;
-        struct radeon_bo_list *lobj;
-        struct radeon_bo *bo;
-        int r;
+        struct radeon_device *rdev = p->rdev;
+        int i, r;
 
-        r = radeon_vm_bo_update(rdev, vm, rdev->ring_tmp_bo.bo, &rdev->ring_tmp_bo.bo->tbo.mem);
-        if (r) {
+        r = radeon_vm_update_page_directory(rdev, vm);
+        if (r)
                 return r;
-        }
-        list_for_each_entry(lobj, &parser->validated, tv.head) {
-                bo = lobj->bo;
-                r = radeon_vm_bo_update(parser->rdev, vm, bo, &bo->tbo.mem);
-                if (r) {
+
+        r = radeon_vm_bo_update(rdev, vm, rdev->ring_tmp_bo.bo,
+                                &rdev->ring_tmp_bo.bo->tbo.mem);
+        if (r)
+                return r;
+
+        for (i = 0; i < p->nrelocs; i++) {
+                struct radeon_bo *bo;
+
+                /* ignore duplicates */
+                if (p->relocs_ptr[i] != &p->relocs[i])
+                        continue;
+
+                bo = p->relocs[i].robj;
+                r = radeon_vm_bo_update(rdev, vm, bo, &bo->tbo.mem);
+                if (r)
                         return r;
-                }
         }
         return 0;
 }
@@ -401,20 +501,13 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
         if (parser->ring == R600_RING_TYPE_UVD_INDEX)
                 radeon_uvd_note_usage(rdev);
 
-        mutex_lock(&rdev->vm_manager.lock);
         mutex_lock(&vm->mutex);
-        r = radeon_vm_alloc_pt(rdev, vm);
-        if (r) {
-                goto out;
-        }
         r = radeon_bo_vm_update_pte(parser, vm);
         if (r) {
                 goto out;
         }
         radeon_cs_sync_rings(parser);
         radeon_semaphore_sync_to(parser->ib.semaphore, vm->fence);
-        radeon_semaphore_sync_to(parser->ib.semaphore,
-                                 radeon_vm_grab_id(rdev, vm, parser->ring));
 
         if ((rdev->family >= CHIP_TAHITI) &&
             (parser->chunk_const_ib_idx != -1)) {
@@ -423,14 +516,8 @@ static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
                 r = radeon_ib_schedule(rdev, &parser->ib, NULL);
         }
 
-        if (!r) {
-                radeon_vm_fence(rdev, vm, parser->ib.fence);
-        }
-
 out:
-        radeon_vm_add_to_lru(rdev, vm);
         mutex_unlock(&vm->mutex);
-        mutex_unlock(&rdev->vm_manager.lock);
         return r;
 }
 
@@ -698,9 +785,9 @@ int radeon_cs_packet_next_reloc(struct radeon_cs_parser *p,
         /* FIXME: we assume reloc size is 4 dwords */
         if (nomm) {
                 *cs_reloc = p->relocs;
-                (*cs_reloc)->lobj.gpu_offset =
+                (*cs_reloc)->gpu_offset =
                         (u64)relocs_chunk->kdata[idx + 3] << 32;
-                (*cs_reloc)->lobj.gpu_offset |= relocs_chunk->kdata[idx + 0];
+                (*cs_reloc)->gpu_offset |= relocs_chunk->kdata[idx + 0];
         } else
                 *cs_reloc = p->relocs_ptr[(idx / 4)];
         return 0;
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 044bc98fb459..835516d2d257 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -1191,14 +1191,12 @@ int radeon_device_init(struct radeon_device *rdev,
         r = radeon_gem_init(rdev);
         if (r)
                 return r;
-        /* initialize vm here */
-        mutex_init(&rdev->vm_manager.lock);
+
         /* Adjust VM size here.
          * Currently set to 4GB ((1 << 20) 4k pages).
          * Max GPUVM size for cayman and SI is 40 bits.
          */
         rdev->vm_manager.max_pfn = 1 << 20;
-        INIT_LIST_HEAD(&rdev->vm_manager.lru_vm);
 
         /* Set asic functions */
         r = radeon_asic_init(rdev);
@@ -1426,7 +1424,7 @@ int radeon_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon)
 
         /* unpin the front buffers */
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-                struct radeon_framebuffer *rfb = to_radeon_framebuffer(crtc->fb);
+                struct radeon_framebuffer *rfb = to_radeon_framebuffer(crtc->primary->fb);
                 struct radeon_bo *robj;
 
                 if (rfb == NULL || rfb->obj == NULL) {
@@ -1445,10 +1443,9 @@ int radeon_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon)
         /* evict vram memory */
         radeon_bo_evict_vram(rdev);
 
-        mutex_lock(&rdev->ring_lock);
         /* wait for gpu to finish processing current batch */
         for (i = 0; i < RADEON_NUM_RINGS; i++) {
-                r = radeon_fence_wait_empty_locked(rdev, i);
+                r = radeon_fence_wait_empty(rdev, i);
                 if (r) {
                         /* delay GPU reset to resume */
                         force_completion = true;
@@ -1457,7 +1454,6 @@ int radeon_suspend_kms(struct drm_device *dev, bool suspend, bool fbcon)
         if (force_completion) {
                 radeon_fence_driver_force_completion(rdev);
         }
-        mutex_unlock(&rdev->ring_lock);
 
         radeon_save_bios_scratch_regs(rdev);
 
@@ -1555,10 +1551,12 @@ int radeon_resume_kms(struct drm_device *dev, bool resume, bool fbcon)
         /* reset hpd state */
         radeon_hpd_init(rdev);
         /* blat the mode back in */
-        drm_helper_resume_force_mode(dev);
-        /* turn on display hw */
-        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
+        if (fbcon) {
+                drm_helper_resume_force_mode(dev);
+                /* turn on display hw */
+                list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
+                        drm_helper_connector_dpms(connector, DRM_MODE_DPMS_ON);
+                }
         }
 
         drm_kms_helper_poll_enable(dev);
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index fbd8b930f2be..386cfa4c194d 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -34,6 +34,8 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
 
+#include <linux/gcd.h>
+
 static void avivo_crtc_load_lut(struct drm_crtc *crtc)
 {
         struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
@@ -369,7 +371,7 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
         work->event = event;
         work->rdev = rdev;
         work->crtc_id = radeon_crtc->crtc_id;
-        old_radeon_fb = to_radeon_framebuffer(crtc->fb);
+        old_radeon_fb = to_radeon_framebuffer(crtc->primary->fb);
         new_radeon_fb = to_radeon_framebuffer(fb);
         /* schedule unpin of the old buffer */
         obj = old_radeon_fb->obj;
@@ -460,7 +462,7 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
         spin_unlock_irqrestore(&dev->event_lock, flags);
 
         /* update crtc fb */
-        crtc->fb = fb;
+        crtc->primary->fb = fb;
 
         r = drm_vblank_get(dev, radeon_crtc->crtc_id);
         if (r) {
@@ -792,6 +794,7 @@ int radeon_ddc_get_modes(struct radeon_connector *radeon_connector)
         if (radeon_connector->edid) {
                 drm_mode_connector_update_edid_property(&radeon_connector->base, radeon_connector->edid);
                 ret = drm_add_edid_modes(&radeon_connector->base, radeon_connector->edid);
+                drm_edid_to_eld(&radeon_connector->base, radeon_connector->edid);
                 return ret;
         }
         drm_mode_connector_update_edid_property(&radeon_connector->base, NULL);
@@ -799,66 +802,57 @@ int radeon_ddc_get_modes(struct radeon_connector *radeon_connector)
 }
 
 /* avivo */
-static void avivo_get_fb_div(struct radeon_pll *pll,
-                             u32 target_clock,
-                             u32 post_div,
-                             u32 ref_div,
-                             u32 *fb_div,
-                             u32 *frac_fb_div)
-{
-        u32 tmp = post_div * ref_div;
 
-        tmp *= target_clock;
-        *fb_div = tmp / pll->reference_freq;
-        *frac_fb_div = tmp % pll->reference_freq;
-
-        if (*fb_div > pll->max_feedback_div)
-                *fb_div = pll->max_feedback_div;
-        else if (*fb_div < pll->min_feedback_div)
-                *fb_div = pll->min_feedback_div;
-}
-
-static u32 avivo_get_post_div(struct radeon_pll *pll,
-                              u32 target_clock)
+/**
+ * avivo_reduce_ratio - fractional number reduction
+ *
+ * @nom: nominator
+ * @den: denominator
+ * @nom_min: minimum value for nominator
+ * @den_min: minimum value for denominator
+ *
+ * Find the greatest common divisor and apply it on both nominator and
+ * denominator, but make nominator and denominator are at least as large
+ * as their minimum values.
+ */
+static void avivo_reduce_ratio(unsigned *nom, unsigned *den,
+                               unsigned nom_min, unsigned den_min)
 {
-        u32 vco, post_div, tmp;
-
-        if (pll->flags & RADEON_PLL_USE_POST_DIV)
-                return pll->post_div;
-
-        if (pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP) {
-                if (pll->flags & RADEON_PLL_IS_LCD)
-                        vco = pll->lcd_pll_out_min;
-                else
-                        vco = pll->pll_out_min;
-        } else {
-                if (pll->flags & RADEON_PLL_IS_LCD)
-                        vco = pll->lcd_pll_out_max;
-                else
-                        vco = pll->pll_out_max;
+        unsigned tmp;
+
+        /* reduce the numbers to a simpler ratio */
+        tmp = gcd(*nom, *den);
+        *nom /= tmp;
+        *den /= tmp;
+
+        /* make sure nominator is large enough */
+        if (*nom < nom_min) {
+                tmp = (nom_min + *nom - 1) / *nom;
+                *nom *= tmp;
+                *den *= tmp;
         }
 
-        post_div = vco / target_clock;
-        tmp = vco % target_clock;
-
-        if (pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP) {
-                if (tmp)
-                        post_div++;
-        } else {
-                if (!tmp)
-                        post_div--;
+        /* make sure the denominator is large enough */
+        if (*den < den_min) {
+                tmp = (den_min + *den - 1) / *den;
+                *nom *= tmp;
+                *den *= tmp;
         }
-
-        if (post_div > pll->max_post_div)
-                post_div = pll->max_post_div;
-        else if (post_div < pll->min_post_div)
-                post_div = pll->min_post_div;
-
-        return post_div;
 }
 
-#define MAX_TOLERANCE 10
-
+/**
+ * radeon_compute_pll_avivo - compute PLL paramaters
+ *
+ * @pll: information about the PLL
+ * @dot_clock_p: resulting pixel clock
+ * fb_div_p: resulting feedback divider
+ * frac_fb_div_p: fractional part of the feedback divider
+ * ref_div_p: resulting reference divider
+ * post_div_p: resulting reference divider
+ *
+ * Try to calculate the PLL parameters to generate the given frequency:
+ * dot_clock = (ref_freq * feedback_div) / (ref_div * post_div)
+ */
 void radeon_compute_pll_avivo(struct radeon_pll *pll,
                               u32 freq,
                               u32 *dot_clock_p,
@@ -867,53 +861,123 @@ void radeon_compute_pll_avivo(struct radeon_pll *pll,
                               u32 *ref_div_p,
                               u32 *post_div_p)
 {
-        u32 target_clock = freq / 10;
-        u32 post_div = avivo_get_post_div(pll, target_clock);
-        u32 ref_div = pll->min_ref_div;
-        u32 fb_div = 0, frac_fb_div = 0, tmp;
+        unsigned fb_div_min, fb_div_max, fb_div;
+        unsigned post_div_min, post_div_max, post_div;
+        unsigned ref_div_min, ref_div_max, ref_div;
+        unsigned post_div_best, diff_best;
+        unsigned nom, den, tmp;
 
-        if (pll->flags & RADEON_PLL_USE_REF_DIV)
-                ref_div = pll->reference_div;
+        /* determine allowed feedback divider range */
+        fb_div_min = pll->min_feedback_div;
+        fb_div_max = pll->max_feedback_div;
 
         if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV) {
-                avivo_get_fb_div(pll, target_clock, post_div, ref_div, &fb_div, &frac_fb_div);
-                frac_fb_div = (100 * frac_fb_div) / pll->reference_freq;
-                if (frac_fb_div >= 5) {
-                        frac_fb_div -= 5;
-                        frac_fb_div = frac_fb_div / 10;
-                        frac_fb_div++;
+                fb_div_min *= 10;
+                fb_div_max *= 10;
+        }
+
+        /* determine allowed ref divider range */
+        if (pll->flags & RADEON_PLL_USE_REF_DIV)
+                ref_div_min = pll->reference_div;
+        else
+                ref_div_min = pll->min_ref_div;
+        ref_div_max = pll->max_ref_div;
+
+        /* determine allowed post divider range */
+        if (pll->flags & RADEON_PLL_USE_POST_DIV) {
+                post_div_min = pll->post_div;
+                post_div_max = pll->post_div;
+        } else {
+                unsigned target_clock = freq / 10;
+                unsigned vco_min, vco_max;
+
+                if (pll->flags & RADEON_PLL_IS_LCD) {
+                        vco_min = pll->lcd_pll_out_min;
+                        vco_max = pll->lcd_pll_out_max;
+                } else {
+                        vco_min = pll->pll_out_min;
+                        vco_max = pll->pll_out_max;
                 }
-                if (frac_fb_div >= 10) {
-                        fb_div++;
-                        frac_fb_div = 0;
+
+                post_div_min = vco_min / target_clock;
+                if ((target_clock * post_div_min) < vco_min)
+                        ++post_div_min;
+                if (post_div_min < pll->min_post_div)
+                        post_div_min = pll->min_post_div;
+
+                post_div_max = vco_max / target_clock;
+                if ((target_clock * post_div_max) > vco_max)
+                        --post_div_max;
+                if (post_div_max > pll->max_post_div)
+                        post_div_max = pll->max_post_div;
+        }
+
+        /* represent the searched ratio as fractional number */
+        nom = pll->flags & RADEON_PLL_USE_FRAC_FB_DIV ? freq : freq / 10;
+        den = pll->reference_freq;
+
+        /* reduce the numbers to a simpler ratio */
+        avivo_reduce_ratio(&nom, &den, fb_div_min, post_div_min);
+
+        /* now search for a post divider */
+        if (pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP)
+                post_div_best = post_div_min;
+        else
+                post_div_best = post_div_max;
+        diff_best = ~0;
+
+        for (post_div = post_div_min; post_div <= post_div_max; ++post_div) {
+                unsigned diff = abs(den - den / post_div * post_div);
+                if (diff < diff_best || (diff == diff_best &&
+                    !(pll->flags & RADEON_PLL_PREFER_MINM_OVER_MAXP))) {
+
+                        post_div_best = post_div;
+                        diff_best = diff;
                 }
+        }
+        post_div = post_div_best;
+
+        /* get matching reference and feedback divider */
+        ref_div = max(den / post_div, 1u);
+        fb_div = nom;
+
+        /* we're almost done, but reference and feedback
+           divider might be to large now */
+
+        tmp = ref_div;
+
+        if (fb_div > fb_div_max) {
+                ref_div = ref_div * fb_div_max / fb_div;
+                fb_div = fb_div_max;
+        }
+
+        if (ref_div > ref_div_max) {
+                ref_div = ref_div_max;
+                fb_div = nom * ref_div_max / tmp;
+        }
+
+        /* reduce the numbers to a simpler ratio once more */
+        /* this also makes sure that the reference divider is large enough */
+        avivo_reduce_ratio(&fb_div, &ref_div, fb_div_min, ref_div_min);
+
+        /* and finally save the result */
+        if (pll->flags & RADEON_PLL_USE_FRAC_FB_DIV) {
+                *fb_div_p = fb_div / 10;
+                *frac_fb_div_p = fb_div % 10;
         } else {
-                while (ref_div <= pll->max_ref_div) {
-                        avivo_get_fb_div(pll, target_clock, post_div, ref_div,
-                                         &fb_div, &frac_fb_div);
-                        if (frac_fb_div >= (pll->reference_freq / 2))
-                                fb_div++;
-                        frac_fb_div = 0;
-                        tmp = (pll->reference_freq * fb_div) / (post_div * ref_div);
-                        tmp = (tmp * 10000) / target_clock;
-
-                        if (tmp > (10000 + MAX_TOLERANCE))
-                                ref_div++;
-                        else if (tmp >= (10000 - MAX_TOLERANCE))
-                                break;
-                        else
-                                ref_div++;
-                }
+                *fb_div_p = fb_div;
+                *frac_fb_div_p = 0;
         }
 
-        *dot_clock_p = ((pll->reference_freq * fb_div * 10) + (pll->reference_freq * frac_fb_div)) /
-                (ref_div * post_div * 10);
-        *fb_div_p = fb_div;
-        *frac_fb_div_p = frac_fb_div;
+        *dot_clock_p = ((pll->reference_freq * *fb_div_p * 10) +
+                        (pll->reference_freq * *frac_fb_div_p)) /
+                       (ref_div * post_div * 10);
         *ref_div_p = ref_div;
         *post_div_p = post_div;
-        DRM_DEBUG_KMS("%d, pll dividers - fb: %d.%d ref: %d, post %d\n",
-                      *dot_clock_p, fb_div, frac_fb_div, ref_div, post_div);
+
+        DRM_DEBUG_KMS("%d - %d, pll dividers - fb: %d.%d ref: %d, post %d\n",
+                      freq, *dot_clock_p, *fb_div_p, *frac_fb_div_p,
+                      ref_div, post_div);
 }
 
 /* pre-avivo */
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index f633c2782170..d0eba48dd74e 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -79,9 +79,11 @@
  *   2.35.0 - Add CIK macrotile mode array query
  *   2.36.0 - Fix CIK DCE tiling setup
  *   2.37.0 - allow GS ring setup on r6xx/r7xx
+ *   2.38.0 - RADEON_GEM_OP (GET_INITIAL_DOMAIN, SET_INITIAL_DOMAIN),
+ *            CIK: 1D and linear tiling modes contain valid PIPE_CONFIG
  */
 #define KMS_DRIVER_MAJOR        2
-#define KMS_DRIVER_MINOR        37
+#define KMS_DRIVER_MINOR        38
 #define KMS_DRIVER_PATCHLEVEL   0
 int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags);
 int radeon_driver_unload_kms(struct drm_device *dev);
diff --git a/drivers/gpu/drm/radeon/radeon_fence.c b/drivers/gpu/drm/radeon/radeon_fence.c
index c37cb79a9489..a77b1c13ea43 100644
--- a/drivers/gpu/drm/radeon/radeon_fence.c
+++ b/drivers/gpu/drm/radeon/radeon_fence.c
@@ -288,7 +288,6 @@ static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq)
  * @rdev: radeon device pointer
  * @target_seq: sequence number(s) we want to wait for
  * @intr: use interruptable sleep
- * @lock_ring: whether the ring should be locked or not
  *
  * Wait for the requested sequence number(s) to be written by any ring
  * (all asics).  Sequnce number array is indexed by ring id.
@@ -299,7 +298,7 @@ static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq)
  * -EDEADLK is returned when a GPU lockup has been detected.
  */
 static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 *target_seq,
-                                 bool intr, bool lock_ring)
+                                 bool intr)
 {
         uint64_t last_seq[RADEON_NUM_RINGS];
         bool signaled;
@@ -358,9 +357,6 @@ static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 *target_seq,
                         if (i != RADEON_NUM_RINGS)
                                 continue;
 
-                        if (lock_ring)
-                                mutex_lock(&rdev->ring_lock);
-
                         for (i = 0; i < RADEON_NUM_RINGS; ++i) {
                                 if (!target_seq[i])
                                         continue;
@@ -378,14 +374,9 @@ static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 *target_seq,
 
                                 /* remember that we need an reset */
                                 rdev->needs_reset = true;
-                                if (lock_ring)
-                                        mutex_unlock(&rdev->ring_lock);
                                 wake_up_all(&rdev->fence_queue);
                                 return -EDEADLK;
                         }
-
-                        if (lock_ring)
-                                mutex_unlock(&rdev->ring_lock);
                 }
         }
         return 0;
@@ -416,7 +407,7 @@ int radeon_fence_wait(struct radeon_fence *fence, bool intr)
         if (seq[fence->ring] == RADEON_FENCE_SIGNALED_SEQ)
                 return 0;
 
-        r = radeon_fence_wait_seq(fence->rdev, seq, intr, true);
+        r = radeon_fence_wait_seq(fence->rdev, seq, intr);
         if (r)
                 return r;
 
@@ -464,7 +455,7 @@ int radeon_fence_wait_any(struct radeon_device *rdev,
         if (num_rings == 0)
                 return -ENOENT;
 
-        r = radeon_fence_wait_seq(rdev, seq, intr, true);
+        r = radeon_fence_wait_seq(rdev, seq, intr);
         if (r) {
                 return r;
         }
@@ -472,37 +463,7 @@ int radeon_fence_wait_any(struct radeon_device *rdev,
 }
 
 /**
- * radeon_fence_wait_locked - wait for a fence to signal
- *
- * @fence: radeon fence object
- *
- * Wait for the requested fence to signal (all asics).
- * Returns 0 if the fence has passed, error for all other cases.
- */
-int radeon_fence_wait_locked(struct radeon_fence *fence)
-{
-        uint64_t seq[RADEON_NUM_RINGS] = {};
-        int r;
-
-        if (fence == NULL) {
-                WARN(1, "Querying an invalid fence : %p !\n", fence);
-                return -EINVAL;
-        }
-
-        seq[fence->ring] = fence->seq;
-        if (seq[fence->ring] == RADEON_FENCE_SIGNALED_SEQ)
-                return 0;
-
-        r = radeon_fence_wait_seq(fence->rdev, seq, false, false);
-        if (r)
-                return r;
-
-        fence->seq = RADEON_FENCE_SIGNALED_SEQ;
-        return 0;
-}
-
-/**
- * radeon_fence_wait_next_locked - wait for the next fence to signal
+ * radeon_fence_wait_next - wait for the next fence to signal
  *
  * @rdev: radeon device pointer
  * @ring: ring index the fence is associated with
@@ -511,7 +472,7 @@ int radeon_fence_wait_locked(struct radeon_fence *fence)
  * Returns 0 if the next fence has passed, error for all other cases.
  * Caller must hold ring lock.
  */
-int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring)
+int radeon_fence_wait_next(struct radeon_device *rdev, int ring)
 {
         uint64_t seq[RADEON_NUM_RINGS] = {};
 
@@ -521,11 +482,11 @@ int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring)
                    already the last emited fence */
                 return -ENOENT;
         }
-        return radeon_fence_wait_seq(rdev, seq, false, false);
+        return radeon_fence_wait_seq(rdev, seq, false);
 }
 
 /**
- * radeon_fence_wait_empty_locked - wait for all fences to signal
+ * radeon_fence_wait_empty - wait for all fences to signal
  *
  * @rdev: radeon device pointer
  * @ring: ring index the fence is associated with
@@ -534,7 +495,7 @@ int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring)
  * Returns 0 if the fences have passed, error for all other cases.
  * Caller must hold ring lock.
  */
-int radeon_fence_wait_empty_locked(struct radeon_device *rdev, int ring)
+int radeon_fence_wait_empty(struct radeon_device *rdev, int ring)
 {
         uint64_t seq[RADEON_NUM_RINGS] = {};
         int r;
@@ -543,7 +504,7 @@ int radeon_fence_wait_empty_locked(struct radeon_device *rdev, int ring)
         if (!seq[ring])
                 return 0;
 
-        r = radeon_fence_wait_seq(rdev, seq, false, false);
+        r = radeon_fence_wait_seq(rdev, seq, false);
         if (r) {
                 if (r == -EDEADLK)
                         return -EDEADLK;
@@ -794,7 +755,7 @@ void radeon_fence_driver_fini(struct radeon_device *rdev)
         for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
                 if (!rdev->fence_drv[ring].initialized)
                         continue;
-                r = radeon_fence_wait_empty_locked(rdev, ring);
+                r = radeon_fence_wait_empty(rdev, ring);
                 if (r) {
                         /* no need to trigger GPU reset as we are unloading */
                         radeon_fence_driver_force_completion(rdev);
diff --git a/drivers/gpu/drm/radeon/radeon_gart.c b/drivers/gpu/drm/radeon/radeon_gart.c
index a8f9b463bf2a..2e723651069b 100644
--- a/drivers/gpu/drm/radeon/radeon_gart.c
+++ b/drivers/gpu/drm/radeon/radeon_gart.c
@@ -28,8 +28,6 @@
 #include <drm/drmP.h>
 #include <drm/radeon_drm.h>
 #include "radeon.h"
-#include "radeon_reg.h"
-#include "radeon_trace.h"
 
 /*
  * GART
@@ -394,959 +392,3 @@ void radeon_gart_fini(struct radeon_device *rdev)
 
         radeon_dummy_page_fini(rdev);
 }
-
-/*
- * GPUVM
- * GPUVM is similar to the legacy gart on older asics, however
- * rather than there being a single global gart table
- * for the entire GPU, there are multiple VM page tables active
- * at any given time.  The VM page tables can contain a mix
- * vram pages and system memory pages and system memory pages
- * can be mapped as snooped (cached system pages) or unsnooped
- * (uncached system pages).
- * Each VM has an ID associated with it and there is a page table
- * associated with each VMID.  When execting a command buffer,
- * the kernel tells the the ring what VMID to use for that command
- * buffer.  VMIDs are allocated dynamically as commands are submitted.
- * The userspace drivers maintain their own address space and the kernel
- * sets up their pages tables accordingly when they submit their
- * command buffers and a VMID is assigned.
- * Cayman/Trinity support up to 8 active VMs at any given time;
- * SI supports 16.
- */
-
-/*
- * vm helpers
- *
- * TODO bind a default page at vm initialization for default address
- */
-
-/**
- * radeon_vm_num_pde - return the number of page directory entries
- *
- * @rdev: radeon_device pointer
- *
- * Calculate the number of page directory entries (cayman+).
- */
-static unsigned radeon_vm_num_pdes(struct radeon_device *rdev)
-{
-        return rdev->vm_manager.max_pfn >> RADEON_VM_BLOCK_SIZE;
-}
-
-/**
- * radeon_vm_directory_size - returns the size of the page directory in bytes
- *
- * @rdev: radeon_device pointer
- *
- * Calculate the size of the page directory in bytes (cayman+).
- */
-static unsigned radeon_vm_directory_size(struct radeon_device *rdev)
-{
-        return RADEON_GPU_PAGE_ALIGN(radeon_vm_num_pdes(rdev) * 8);
-}
-
-/**
- * radeon_vm_manager_init - init the vm manager
- *
- * @rdev: radeon_device pointer
- *
- * Init the vm manager (cayman+).
- * Returns 0 for success, error for failure.
- */
-int radeon_vm_manager_init(struct radeon_device *rdev)
-{
-        struct radeon_vm *vm;
-        struct radeon_bo_va *bo_va;
-        int r;
-        unsigned size;
-
-        if (!rdev->vm_manager.enabled) {
-                /* allocate enough for 2 full VM pts */
-                size = radeon_vm_directory_size(rdev);
-                size += rdev->vm_manager.max_pfn * 8;
-                size *= 2;
-                r = radeon_sa_bo_manager_init(rdev, &rdev->vm_manager.sa_manager,
-                                              RADEON_GPU_PAGE_ALIGN(size),
-                                              RADEON_VM_PTB_ALIGN_SIZE,
-                                              RADEON_GEM_DOMAIN_VRAM);
-                if (r) {
-                        dev_err(rdev->dev, "failed to allocate vm bo (%dKB)\n",
-                                (rdev->vm_manager.max_pfn * 8) >> 10);
-                        return r;
-                }
-
-                r = radeon_asic_vm_init(rdev);
-                if (r)
-                        return r;
-
-                rdev->vm_manager.enabled = true;
-
-                r = radeon_sa_bo_manager_start(rdev, &rdev->vm_manager.sa_manager);
-                if (r)
-                        return r;
-        }
-
-        /* restore page table */
-        list_for_each_entry(vm, &rdev->vm_manager.lru_vm, list) {
-                if (vm->page_directory == NULL)
-                        continue;
-
-                list_for_each_entry(bo_va, &vm->va, vm_list) {
-                        bo_va->valid = false;
-                }
-        }
-        return 0;
-}
-
-/**
- * radeon_vm_free_pt - free the page table for a specific vm
- *
- * @rdev: radeon_device pointer
- * @vm: vm to unbind
- *
- * Free the page table of a specific vm (cayman+).
- *
- * Global and local mutex must be lock!
- */
-static void radeon_vm_free_pt(struct radeon_device *rdev,
-                                    struct radeon_vm *vm)
-{
-        struct radeon_bo_va *bo_va;
-        int i;
-
-        if (!vm->page_directory)
-                return;
-
-        list_del_init(&vm->list);
-        radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence);
-
-        list_for_each_entry(bo_va, &vm->va, vm_list) {
-                bo_va->valid = false;
-        }
-
-        if (vm->page_tables == NULL)
-                return;
-
-        for (i = 0; i < radeon_vm_num_pdes(rdev); i++)
-                radeon_sa_bo_free(rdev, &vm->page_tables[i], vm->fence);
-
-        kfree(vm->page_tables);
-}
-
-/**
- * radeon_vm_manager_fini - tear down the vm manager
- *
- * @rdev: radeon_device pointer
- *
- * Tear down the VM manager (cayman+).
- */
-void radeon_vm_manager_fini(struct radeon_device *rdev)
-{
-        struct radeon_vm *vm, *tmp;
-        int i;
-
-        if (!rdev->vm_manager.enabled)
-                return;
-
-        mutex_lock(&rdev->vm_manager.lock);
-        /* free all allocated page tables */
-        list_for_each_entry_safe(vm, tmp, &rdev->vm_manager.lru_vm, list) {
-                mutex_lock(&vm->mutex);
-                radeon_vm_free_pt(rdev, vm);
-                mutex_unlock(&vm->mutex);
-        }
-        for (i = 0; i < RADEON_NUM_VM; ++i) {
-                radeon_fence_unref(&rdev->vm_manager.active[i]);
-        }
-        radeon_asic_vm_fini(rdev);
-        mutex_unlock(&rdev->vm_manager.lock);
-
-        radeon_sa_bo_manager_suspend(rdev, &rdev->vm_manager.sa_manager);
-        radeon_sa_bo_manager_fini(rdev, &rdev->vm_manager.sa_manager);
-        rdev->vm_manager.enabled = false;
-}
-
-/**
- * radeon_vm_evict - evict page table to make room for new one
- *
- * @rdev: radeon_device pointer
- * @vm: VM we want to allocate something for
- *
- * Evict a VM from the lru, making sure that it isn't @vm. (cayman+).
- * Returns 0 for success, -ENOMEM for failure.
- *
- * Global and local mutex must be locked!
- */
-static int radeon_vm_evict(struct radeon_device *rdev, struct radeon_vm *vm)
-{
-        struct radeon_vm *vm_evict;
-
-        if (list_empty(&rdev->vm_manager.lru_vm))
-                return -ENOMEM;
-
-        vm_evict = list_first_entry(&rdev->vm_manager.lru_vm,
-                                    struct radeon_vm, list);
-        if (vm_evict == vm)
-                return -ENOMEM;
-
-        mutex_lock(&vm_evict->mutex);
-        radeon_vm_free_pt(rdev, vm_evict);
-        mutex_unlock(&vm_evict->mutex);
-        return 0;
-}
-
-/**
- * radeon_vm_alloc_pt - allocates a page table for a VM
- *
- * @rdev: radeon_device pointer
- * @vm: vm to bind
- *
- * Allocate a page table for the requested vm (cayman+).
- * Returns 0 for success, error for failure.
- *
- * Global and local mutex must be locked!
- */
-int radeon_vm_alloc_pt(struct radeon_device *rdev, struct radeon_vm *vm)
-{
-        unsigned pd_size, pd_entries, pts_size;
-        struct radeon_ib ib;
-        int r;
-
-        if (vm == NULL) {
-                return -EINVAL;
-        }
-
-        if (vm->page_directory != NULL) {
-                return 0;
-        }
-
-        pd_size = radeon_vm_directory_size(rdev);
-        pd_entries = radeon_vm_num_pdes(rdev);
-
-retry:
-        r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager,
-                             &vm->page_directory, pd_size,
-                             RADEON_VM_PTB_ALIGN_SIZE, false);
-        if (r == -ENOMEM) {
-                r = radeon_vm_evict(rdev, vm);
-                if (r)
-                        return r;
-                goto retry;
-
-        } else if (r) {
-                return r;
-        }
-
-        vm->pd_gpu_addr = radeon_sa_bo_gpu_addr(vm->page_directory);
-
-        /* Initially clear the page directory */
-        r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib,
-                          NULL, pd_entries * 2 + 64);
-        if (r) {
-                radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence);
-                return r;
-        }
-
-        ib.length_dw = 0;
-
-        radeon_asic_vm_set_page(rdev, &ib, vm->pd_gpu_addr,
-                                0, pd_entries, 0, 0);
-
-        radeon_semaphore_sync_to(ib.semaphore, vm->fence);
-        r = radeon_ib_schedule(rdev, &ib, NULL);
-        if (r) {
-                radeon_ib_free(rdev, &ib);
-                radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence);
-                return r;
-        }
-        radeon_fence_unref(&vm->fence);
-        vm->fence = radeon_fence_ref(ib.fence);
-        radeon_ib_free(rdev, &ib);
-        radeon_fence_unref(&vm->last_flush);
-
-        /* allocate page table array */
-        pts_size = radeon_vm_num_pdes(rdev) * sizeof(struct radeon_sa_bo *);
-        vm->page_tables = kzalloc(pts_size, GFP_KERNEL);
-
-        if (vm->page_tables == NULL) {
-                DRM_ERROR("Cannot allocate memory for page table array\n");
-                radeon_sa_bo_free(rdev, &vm->page_directory, vm->fence);
-                return -ENOMEM;
-        }
-
-        return 0;
-}
-
-/**
- * radeon_vm_add_to_lru - add VMs page table to LRU list
- *
- * @rdev: radeon_device pointer
- * @vm: vm to add to LRU
- *
- * Add the allocated page table to the LRU list (cayman+).
- *
- * Global mutex must be locked!
- */
-void radeon_vm_add_to_lru(struct radeon_device *rdev, struct radeon_vm *vm)
-{
-        list_del_init(&vm->list);
-        list_add_tail(&vm->list, &rdev->vm_manager.lru_vm);
-}
-
-/**
- * radeon_vm_grab_id - allocate the next free VMID
- *
- * @rdev: radeon_device pointer
- * @vm: vm to allocate id for
- * @ring: ring we want to submit job to
- *
- * Allocate an id for the vm (cayman+).
- * Returns the fence we need to sync to (if any).
- *
- * Global and local mutex must be locked!
- */
-struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev,
-                                       struct radeon_vm *vm, int ring)
-{
-        struct radeon_fence *best[RADEON_NUM_RINGS] = {};
-        unsigned choices[2] = {};
-        unsigned i;
-
-        /* check if the id is still valid */
-        if (vm->last_id_use && vm->last_id_use == rdev->vm_manager.active[vm->id])
-                return NULL;
-
-        /* we definately need to flush */
-        radeon_fence_unref(&vm->last_flush);
-
-        /* skip over VMID 0, since it is the system VM */
-        for (i = 1; i < rdev->vm_manager.nvm; ++i) {
-                struct radeon_fence *fence = rdev->vm_manager.active[i];
-
-                if (fence == NULL) {
-                        /* found a free one */
-                        vm->id = i;
-                        trace_radeon_vm_grab_id(vm->id, ring);
-                        return NULL;
-                }
-
-                if (radeon_fence_is_earlier(fence, best[fence->ring])) {
-                        best[fence->ring] = fence;
-                        choices[fence->ring == ring ? 0 : 1] = i;
-                }
-        }
-
-        for (i = 0; i < 2; ++i) {
-                if (choices[i]) {
-                        vm->id = choices[i];
-                        trace_radeon_vm_grab_id(vm->id, ring);
-                        return rdev->vm_manager.active[choices[i]];
-                }
-        }
-
-        /* should never happen */
-        BUG();
-        return NULL;
-}
-
-/**
- * radeon_vm_fence - remember fence for vm
- *
- * @rdev: radeon_device pointer
- * @vm: vm we want to fence
- * @fence: fence to remember
- *
- * Fence the vm (cayman+).
- * Set the fence used to protect page table and id.
- *
- * Global and local mutex must be locked!
- */
-void radeon_vm_fence(struct radeon_device *rdev,
-                     struct radeon_vm *vm,
-                     struct radeon_fence *fence)
-{
-        radeon_fence_unref(&rdev->vm_manager.active[vm->id]);
-        rdev->vm_manager.active[vm->id] = radeon_fence_ref(fence);
-
-        radeon_fence_unref(&vm->fence);
-        vm->fence = radeon_fence_ref(fence);
-
-        radeon_fence_unref(&vm->last_id_use);
-        vm->last_id_use = radeon_fence_ref(fence);
-}
-
-/**
- * radeon_vm_bo_find - find the bo_va for a specific vm & bo
- *
- * @vm: requested vm
- * @bo: requested buffer object
- *
- * Find @bo inside the requested vm (cayman+).
- * Search inside the @bos vm list for the requested vm
- * Returns the found bo_va or NULL if none is found
- *
- * Object has to be reserved!
- */
-struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm,
-                                       struct radeon_bo *bo)
-{
-        struct radeon_bo_va *bo_va;
-
-        list_for_each_entry(bo_va, &bo->va, bo_list) {
-                if (bo_va->vm == vm) {
-                        return bo_va;
-                }
-        }
-        return NULL;
-}
-
-/**
- * radeon_vm_bo_add - add a bo to a specific vm
- *
- * @rdev: radeon_device pointer
- * @vm: requested vm
- * @bo: radeon buffer object
- *
- * Add @bo into the requested vm (cayman+).
- * Add @bo to the list of bos associated with the vm
- * Returns newly added bo_va or NULL for failure
- *
- * Object has to be reserved!
- */
-struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev,
-                                      struct radeon_vm *vm,
-                                      struct radeon_bo *bo)
-{
-        struct radeon_bo_va *bo_va;
-
-        bo_va = kzalloc(sizeof(struct radeon_bo_va), GFP_KERNEL);
-        if (bo_va == NULL) {
-                return NULL;
-        }
-        bo_va->vm = vm;
-        bo_va->bo = bo;
-        bo_va->soffset = 0;
-        bo_va->eoffset = 0;
-        bo_va->flags = 0;
-        bo_va->valid = false;
-        bo_va->ref_count = 1;
-        INIT_LIST_HEAD(&bo_va->bo_list);
-        INIT_LIST_HEAD(&bo_va->vm_list);
-
-        mutex_lock(&vm->mutex);
-        list_add(&bo_va->vm_list, &vm->va);
-        list_add_tail(&bo_va->bo_list, &bo->va);
-        mutex_unlock(&vm->mutex);
-
-        return bo_va;
-}
-
-/**
- * radeon_vm_bo_set_addr - set bos virtual address inside a vm
- *
- * @rdev: radeon_device pointer
- * @bo_va: bo_va to store the address
- * @soffset: requested offset of the buffer in the VM address space
- * @flags: attributes of pages (read/write/valid/etc.)
- *
- * Set offset of @bo_va (cayman+).
- * Validate and set the offset requested within the vm address space.
- * Returns 0 for success, error for failure.
- *
- * Object has to be reserved!
- */
-int radeon_vm_bo_set_addr(struct radeon_device *rdev,
-                          struct radeon_bo_va *bo_va,
-                          uint64_t soffset,
-                          uint32_t flags)
-{
-        uint64_t size = radeon_bo_size(bo_va->bo);
-        uint64_t eoffset, last_offset = 0;
-        struct radeon_vm *vm = bo_va->vm;
-        struct radeon_bo_va *tmp;
-        struct list_head *head;
-        unsigned last_pfn;
-
-        if (soffset) {
-                /* make sure object fit at this offset */
-                eoffset = soffset + size;
-                if (soffset >= eoffset) {
-                        return -EINVAL;
-                }
-
-                last_pfn = eoffset / RADEON_GPU_PAGE_SIZE;
-                if (last_pfn > rdev->vm_manager.max_pfn) {
-                        dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n",
-                                last_pfn, rdev->vm_manager.max_pfn);
-                        return -EINVAL;
-                }
-
-        } else {
-                eoffset = last_pfn = 0;
-        }
-
-        mutex_lock(&vm->mutex);
-        head = &vm->va;
-        last_offset = 0;
-        list_for_each_entry(tmp, &vm->va, vm_list) {
-                if (bo_va == tmp) {
-                        /* skip over currently modified bo */
-                        continue;
-                }
-
-                if (soffset >= last_offset && eoffset <= tmp->soffset) {
-                        /* bo can be added before this one */
-                        break;
-                }
-                if (eoffset > tmp->soffset && soffset < tmp->eoffset) {
-                        /* bo and tmp overlap, invalid offset */
-                        dev_err(rdev->dev, "bo %p va 0x%08X conflict with (bo %p 0x%08X 0x%08X)\n",
-                                bo_va->bo, (unsigned)bo_va->soffset, tmp->bo,
-                                (unsigned)tmp->soffset, (unsigned)tmp->eoffset);
-                        mutex_unlock(&vm->mutex);
-                        return -EINVAL;
-                }
-                last_offset = tmp->eoffset;
-                head = &tmp->vm_list;
-        }
-
-        bo_va->soffset = soffset;
-        bo_va->eoffset = eoffset;
-        bo_va->flags = flags;
-        bo_va->valid = false;
-        list_move(&bo_va->vm_list, head);
-
-        mutex_unlock(&vm->mutex);
-        return 0;
-}
-
-/**
- * radeon_vm_map_gart - get the physical address of a gart page
- *
- * @rdev: radeon_device pointer
- * @addr: the unmapped addr
- *
- * Look up the physical address of the page that the pte resolves
- * to (cayman+).
- * Returns the physical address of the page.
- */
-uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr)
-{
-        uint64_t result;
-
-        /* page table offset */
-        result = rdev->gart.pages_addr[addr >> PAGE_SHIFT];
-
-        /* in case cpu page size != gpu page size*/
-        result |= addr & (~PAGE_MASK);
-
-        return result;
-}
-
-/**
- * radeon_vm_page_flags - translate page flags to what the hw uses
- *
- * @flags: flags comming from userspace
- *
- * Translate the flags the userspace ABI uses to hw flags.
- */
-static uint32_t radeon_vm_page_flags(uint32_t flags)
-{
-        uint32_t hw_flags = 0;
-        hw_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_PTE_VALID : 0;
-        hw_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0;
-        hw_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0;
-        if (flags & RADEON_VM_PAGE_SYSTEM) {
-                hw_flags |= R600_PTE_SYSTEM;
-                hw_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0;
-        }
-        return hw_flags;
-}
-
-/**
- * radeon_vm_update_pdes - make sure that page directory is valid
- *
- * @rdev: radeon_device pointer
- * @vm: requested vm
- * @start: start of GPU address range
- * @end: end of GPU address range
- *
- * Allocates new page tables if necessary
- * and updates the page directory (cayman+).
- * Returns 0 for success, error for failure.
- *
- * Global and local mutex must be locked!
- */
-static int radeon_vm_update_pdes(struct radeon_device *rdev,
-                                 struct radeon_vm *vm,
-                                 struct radeon_ib *ib,
-                                 uint64_t start, uint64_t end)
-{
-        static const uint32_t incr = RADEON_VM_PTE_COUNT * 8;
-
-        uint64_t last_pde = ~0, last_pt = ~0;
-        unsigned count = 0;
-        uint64_t pt_idx;
-        int r;
-
-        start = (start / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE;
-        end = (end / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE;
-
-        /* walk over the address space and update the page directory */
-        for (pt_idx = start; pt_idx <= end; ++pt_idx) {
-                uint64_t pde, pt;
-
-                if (vm->page_tables[pt_idx])
-                        continue;
-
-retry:
-                r = radeon_sa_bo_new(rdev, &rdev->vm_manager.sa_manager,
-                                     &vm->page_tables[pt_idx],
-                                     RADEON_VM_PTE_COUNT * 8,
-                                     RADEON_GPU_PAGE_SIZE, false);
-
-                if (r == -ENOMEM) {
-                        r = radeon_vm_evict(rdev, vm);
-                        if (r)
-                                return r;
-                        goto retry;
-                } else if (r) {
-                        return r;
-                }
-
-                pde = vm->pd_gpu_addr + pt_idx * 8;
-
-                pt = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]);
-
-                if (((last_pde + 8 * count) != pde) ||
-                    ((last_pt + incr * count) != pt)) {
-
-                        if (count) {
-                                radeon_asic_vm_set_page(rdev, ib, last_pde,
-                                                        last_pt, count, incr,
-                                                        R600_PTE_VALID);
-
-                                count *= RADEON_VM_PTE_COUNT;
-                                radeon_asic_vm_set_page(rdev, ib, last_pt, 0,
-                                                        count, 0, 0);
-                        }
-
-                        count = 1;
-                        last_pde = pde;
-                        last_pt = pt;
-                } else {
-                        ++count;
-                }
-        }
-
-        if (count) {
-                radeon_asic_vm_set_page(rdev, ib, last_pde, last_pt, count,
-                                        incr, R600_PTE_VALID);
-
-                count *= RADEON_VM_PTE_COUNT;
-                radeon_asic_vm_set_page(rdev, ib, last_pt, 0,
-                                        count, 0, 0);
-        }
-
-        return 0;
-}
-
-/**
- * radeon_vm_update_ptes - make sure that page tables are valid
- *
- * @rdev: radeon_device pointer
- * @vm: requested vm
- * @start: start of GPU address range
- * @end: end of GPU address range
- * @dst: destination address to map to
- * @flags: mapping flags
- *
- * Update the page tables in the range @start - @end (cayman+).
- *
- * Global and local mutex must be locked!
- */
-static void radeon_vm_update_ptes(struct radeon_device *rdev,
-                                  struct radeon_vm *vm,
-                                  struct radeon_ib *ib,
-                                  uint64_t start, uint64_t end,
-                                  uint64_t dst, uint32_t flags)
-{
-        static const uint64_t mask = RADEON_VM_PTE_COUNT - 1;
-
-        uint64_t last_pte = ~0, last_dst = ~0;
-        unsigned count = 0;
-        uint64_t addr;
-
-        start = start / RADEON_GPU_PAGE_SIZE;
-        end = end / RADEON_GPU_PAGE_SIZE;
-
-        /* walk over the address space and update the page tables */
-        for (addr = start; addr < end; ) {
-                uint64_t pt_idx = addr >> RADEON_VM_BLOCK_SIZE;
-                unsigned nptes;
-                uint64_t pte;
-
-                if ((addr & ~mask) == (end & ~mask))
-                        nptes = end - addr;
-                else
-                        nptes = RADEON_VM_PTE_COUNT - (addr & mask);
-
-                pte = radeon_sa_bo_gpu_addr(vm->page_tables[pt_idx]);
-                pte += (addr & mask) * 8;
-
-                if ((last_pte + 8 * count) != pte) {
-
-                        if (count) {
-                                radeon_asic_vm_set_page(rdev, ib, last_pte,
-                                                        last_dst, count,
-                                                        RADEON_GPU_PAGE_SIZE,
-                                                        flags);
-                        }
-
-                        count = nptes;
-                        last_pte = pte;
-                        last_dst = dst;
-                } else {
-                        count += nptes;
-                }
-
-                addr += nptes;
-                dst += nptes * RADEON_GPU_PAGE_SIZE;
-        }
-
-        if (count) {
-                radeon_asic_vm_set_page(rdev, ib, last_pte,
-                                        last_dst, count,
-                                        RADEON_GPU_PAGE_SIZE, flags);
-        }
-}
-
-/**
- * radeon_vm_bo_update - map a bo into the vm page table
- *
- * @rdev: radeon_device pointer
- * @vm: requested vm
- * @bo: radeon buffer object
- * @mem: ttm mem
- *
- * Fill in the page table entries for @bo (cayman+).
- * Returns 0 for success, -EINVAL for failure.
- *
- * Object have to be reserved & global and local mutex must be locked!
- */
-int radeon_vm_bo_update(struct radeon_device *rdev,
-                        struct radeon_vm *vm,
-                        struct radeon_bo *bo,
-                        struct ttm_mem_reg *mem)
-{
-        struct radeon_ib ib;
-        struct radeon_bo_va *bo_va;
-        unsigned nptes, npdes, ndw;
-        uint64_t addr;
-        int r;
-
-        /* nothing to do if vm isn't bound */
-        if (vm->page_directory == NULL)
-                return 0;
-
-        bo_va = radeon_vm_bo_find(vm, bo);
-        if (bo_va == NULL) {
-                dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm);
-                return -EINVAL;
-        }
-
-        if (!bo_va->soffset) {
-                dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n",
-                        bo, vm);
-                return -EINVAL;
-        }
-
-        if ((bo_va->valid && mem) || (!bo_va->valid && mem == NULL))
-                return 0;
-
-        bo_va->flags &= ~RADEON_VM_PAGE_VALID;
-        bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM;
-        if (mem) {
-                addr = mem->start << PAGE_SHIFT;
-                if (mem->mem_type != TTM_PL_SYSTEM) {
-                        bo_va->flags |= RADEON_VM_PAGE_VALID;
-                        bo_va->valid = true;
-                }
-                if (mem->mem_type == TTM_PL_TT) {
-                        bo_va->flags |= RADEON_VM_PAGE_SYSTEM;
-                } else {
-                        addr += rdev->vm_manager.vram_base_offset;
-                }
-        } else {
-                addr = 0;
-                bo_va->valid = false;
-        }
-
-        trace_radeon_vm_bo_update(bo_va);
-
-        nptes = radeon_bo_ngpu_pages(bo);
-
-        /* assume two extra pdes in case the mapping overlaps the borders */
-        npdes = (nptes >> RADEON_VM_BLOCK_SIZE) + 2;
-
-        /* padding, etc. */
-        ndw = 64;
-
-        if (RADEON_VM_BLOCK_SIZE > 11)
-                /* reserve space for one header for every 2k dwords */
-                ndw += (nptes >> 11) * 4;
-        else
-                /* reserve space for one header for
-                    every (1 << BLOCK_SIZE) entries */
-                ndw += (nptes >> RADEON_VM_BLOCK_SIZE) * 4;
-
-        /* reserve space for pte addresses */
-        ndw += nptes * 2;
-
-        /* reserve space for one header for every 2k dwords */
-        ndw += (npdes >> 11) * 4;
-
-        /* reserve space for pde addresses */
-        ndw += npdes * 2;
-
-        /* reserve space for clearing new page tables */
-        ndw += npdes * 2 * RADEON_VM_PTE_COUNT;
-
-        /* update too big for an IB */
-        if (ndw > 0xfffff)
-                return -ENOMEM;
-
-        r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4);
-        if (r)
-                return r;
-        ib.length_dw = 0;
-
-        r = radeon_vm_update_pdes(rdev, vm, &ib, bo_va->soffset, bo_va->eoffset);
-        if (r) {
-                radeon_ib_free(rdev, &ib);
-                return r;
-        }
-
-        radeon_vm_update_ptes(rdev, vm, &ib, bo_va->soffset, bo_va->eoffset,
-                              addr, radeon_vm_page_flags(bo_va->flags));
-
-        radeon_semaphore_sync_to(ib.semaphore, vm->fence);
-        r = radeon_ib_schedule(rdev, &ib, NULL);
-        if (r) {
-                radeon_ib_free(rdev, &ib);
-                return r;
-        }
-        radeon_fence_unref(&vm->fence);
-        vm->fence = radeon_fence_ref(ib.fence);
-        radeon_ib_free(rdev, &ib);
-        radeon_fence_unref(&vm->last_flush);
-
-        return 0;
-}
-
-/**
- * radeon_vm_bo_rmv - remove a bo to a specific vm
- *
- * @rdev: radeon_device pointer
- * @bo_va: requested bo_va
- *
- * Remove @bo_va->bo from the requested vm (cayman+).
- * Remove @bo_va->bo from the list of bos associated with the bo_va->vm and
- * remove the ptes for @bo_va in the page table.
- * Returns 0 for success.
- *
- * Object have to be reserved!
- */
-int radeon_vm_bo_rmv(struct radeon_device *rdev,
-                     struct radeon_bo_va *bo_va)
-{
-        int r = 0;
-
-        mutex_lock(&rdev->vm_manager.lock);
-        mutex_lock(&bo_va->vm->mutex);
-        if (bo_va->soffset) {
-                r = radeon_vm_bo_update(rdev, bo_va->vm, bo_va->bo, NULL);
-        }
-        mutex_unlock(&rdev->vm_manager.lock);
-        list_del(&bo_va->vm_list);
-        mutex_unlock(&bo_va->vm->mutex);
-        list_del(&bo_va->bo_list);
-
-        kfree(bo_va);
-        return r;
-}
-
-/**
- * radeon_vm_bo_invalidate - mark the bo as invalid
- *
- * @rdev: radeon_device pointer
- * @vm: requested vm
- * @bo: radeon buffer object
- *
- * Mark @bo as invalid (cayman+).
- */
-void radeon_vm_bo_invalidate(struct radeon_device *rdev,
-                             struct radeon_bo *bo)
-{
-        struct radeon_bo_va *bo_va;
-
-        list_for_each_entry(bo_va, &bo->va, bo_list) {
-                bo_va->valid = false;
-        }
-}
-
-/**
- * radeon_vm_init - initialize a vm instance
- *
- * @rdev: radeon_device pointer
- * @vm: requested vm
- *
- * Init @vm fields (cayman+).
- */
-void radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm)
-{
-        vm->id = 0;
-        vm->fence = NULL;
-        vm->last_flush = NULL;
-        vm->last_id_use = NULL;
-        mutex_init(&vm->mutex);
-        INIT_LIST_HEAD(&vm->list);
-        INIT_LIST_HEAD(&vm->va);
-}
-
-/**
- * radeon_vm_fini - tear down a vm instance
- *
- * @rdev: radeon_device pointer
- * @vm: requested vm
- *
- * Tear down @vm (cayman+).
- * Unbind the VM and remove all bos from the vm bo list
- */
-void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm)
-{
-        struct radeon_bo_va *bo_va, *tmp;
-        int r;
-
-        mutex_lock(&rdev->vm_manager.lock);
-        mutex_lock(&vm->mutex);
-        radeon_vm_free_pt(rdev, vm);
-        mutex_unlock(&rdev->vm_manager.lock);
-
-        if (!list_empty(&vm->va)) {
-                dev_err(rdev->dev, "still active bo inside vm\n");
-        }
-        list_for_each_entry_safe(bo_va, tmp, &vm->va, vm_list) {
-                list_del_init(&bo_va->vm_list);
-                r = radeon_bo_reserve(bo_va->bo, false);
-                if (!r) {
-                        list_del_init(&bo_va->bo_list);
-                        radeon_bo_unreserve(bo_va->bo);
-                        kfree(bo_va);
-                }
-        }
-        radeon_fence_unref(&vm->fence);
-        radeon_fence_unref(&vm->last_flush);
-        radeon_fence_unref(&vm->last_id_use);
-        mutex_unlock(&vm->mutex);
-}
diff --git a/drivers/gpu/drm/radeon/radeon_gem.c b/drivers/gpu/drm/radeon/radeon_gem.c
index b96c819024b3..d09650c1d720 100644
--- a/drivers/gpu/drm/radeon/radeon_gem.c
+++ b/drivers/gpu/drm/radeon/radeon_gem.c
@@ -344,18 +344,7 @@ int radeon_gem_busy_ioctl(struct drm_device *dev, void *data,
         }
         robj = gem_to_radeon_bo(gobj);
         r = radeon_bo_wait(robj, &cur_placement, true);
-        switch (cur_placement) {
-        case TTM_PL_VRAM:
-                args->domain = RADEON_GEM_DOMAIN_VRAM;
-                break;
-        case TTM_PL_TT:
-                args->domain = RADEON_GEM_DOMAIN_GTT;
-                break;
-        case TTM_PL_SYSTEM:
-                args->domain = RADEON_GEM_DOMAIN_CPU;
-        default:
-                break;
-        }
+        args->domain = radeon_mem_type_to_domain(cur_placement);
         drm_gem_object_unreference_unlocked(gobj);
         r = radeon_gem_handle_lockup(rdev, r);
         return r;
@@ -533,6 +522,42 @@ out:
         return r;
 }
 
+int radeon_gem_op_ioctl(struct drm_device *dev, void *data,
+                        struct drm_file *filp)
+{
+        struct drm_radeon_gem_op *args = data;
+        struct drm_gem_object *gobj;
+        struct radeon_bo *robj;
+        int r;
+
+        gobj = drm_gem_object_lookup(dev, filp, args->handle);
+        if (gobj == NULL) {
+                return -ENOENT;
+        }
+        robj = gem_to_radeon_bo(gobj);
+        r = radeon_bo_reserve(robj, false);
+        if (unlikely(r))
+                goto out;
+
+        switch (args->op) {
+        case RADEON_GEM_OP_GET_INITIAL_DOMAIN:
+                args->value = robj->initial_domain;
+                break;
+        case RADEON_GEM_OP_SET_INITIAL_DOMAIN:
+                robj->initial_domain = args->value & (RADEON_GEM_DOMAIN_VRAM |
+                                                      RADEON_GEM_DOMAIN_GTT |
+                                                      RADEON_GEM_DOMAIN_CPU);
+                break;
+        default:
+                r = -EINVAL;
+        }
+
+        radeon_bo_unreserve(robj);
+out:
+        drm_gem_object_unreference_unlocked(gobj);
+        return r;
+}
+
 int radeon_mode_dumb_create(struct drm_file *file_priv,
                             struct drm_device *dev,
                             struct drm_mode_create_dumb *args)
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index 66ed3ea71440..3e49342a20e6 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -441,6 +441,9 @@ static int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file
                 case RADEON_CS_RING_UVD:
                         *value = rdev->ring[R600_RING_TYPE_UVD_INDEX].ready;
                         break;
+                case RADEON_CS_RING_VCE:
+                        *value = rdev->ring[TN_RING_TYPE_VCE1_INDEX].ready;
+                        break;
                 default:
                         return -EINVAL;
                 }
@@ -485,6 +488,27 @@ static int radeon_info_ioctl(struct drm_device *dev, void *data, struct drm_file
                 else
                         *value = rdev->pm.default_sclk * 10;
                 break;
+        case RADEON_INFO_VCE_FW_VERSION:
+                *value = rdev->vce.fw_version;
+                break;
+        case RADEON_INFO_VCE_FB_VERSION:
+                *value = rdev->vce.fb_version;
+                break;
+        case RADEON_INFO_NUM_BYTES_MOVED:
+                value = (uint32_t*)&value64;
+                value_size = sizeof(uint64_t);
+                value64 = atomic64_read(&rdev->num_bytes_moved);
+                break;
+        case RADEON_INFO_VRAM_USAGE:
+                value = (uint32_t*)&value64;
+                value_size = sizeof(uint64_t);
+                value64 = atomic64_read(&rdev->vram_usage);
+                break;
+        case RADEON_INFO_GTT_USAGE:
+                value = (uint32_t*)&value64;
+                value_size = sizeof(uint64_t);
+                value64 = atomic64_read(&rdev->gtt_usage);
+                break;
         default:
                 DRM_DEBUG_KMS("Invalid request %d\n", info->request);
                 return -EINVAL;
@@ -543,7 +567,9 @@ int radeon_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
                         return -ENOMEM;
                 }
 
-                radeon_vm_init(rdev, &fpriv->vm);
+                r = radeon_vm_init(rdev, &fpriv->vm);
+                if (r)
+                        return r;
 
                 r = radeon_bo_reserve(rdev->ring_tmp_bo.bo, false);
                 if (r)
@@ -624,6 +650,7 @@ void radeon_driver_preclose_kms(struct drm_device *dev,
         if (rdev->cmask_filp == file_priv)
                 rdev->cmask_filp = NULL;
         radeon_uvd_free_handles(rdev, file_priv);
+        radeon_vce_free_handles(rdev, file_priv);
 }
 
 /*
@@ -818,5 +845,6 @@ const struct drm_ioctl_desc radeon_ioctls_kms[] = {
         DRM_IOCTL_DEF_DRV(RADEON_GEM_GET_TILING, radeon_gem_get_tiling_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
         DRM_IOCTL_DEF_DRV(RADEON_GEM_BUSY, radeon_gem_busy_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
         DRM_IOCTL_DEF_DRV(RADEON_GEM_VA, radeon_gem_va_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(RADEON_GEM_OP, radeon_gem_op_ioctl, DRM_AUTH|DRM_UNLOCKED|DRM_RENDER_ALLOW),
 };
 int radeon_max_kms_ioctl = DRM_ARRAY_SIZE(radeon_ioctls_kms);
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
index 0b158f98d287..cafb1ccf2ec3 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
@@ -385,7 +385,7 @@ int radeon_crtc_do_set_base(struct drm_crtc *crtc,
 
         DRM_DEBUG_KMS("\n");
         /* no fb bound */
-        if (!atomic && !crtc->fb) {
+        if (!atomic && !crtc->primary->fb) {
                 DRM_DEBUG_KMS("No FB bound\n");
                 return 0;
         }
@@ -395,8 +395,8 @@ int radeon_crtc_do_set_base(struct drm_crtc *crtc,
                 target_fb = fb;
         }
         else {
-                radeon_fb = to_radeon_framebuffer(crtc->fb);
-                target_fb = crtc->fb;
+                radeon_fb = to_radeon_framebuffer(crtc->primary->fb);
+                target_fb = crtc->primary->fb;
         }
 
         switch (target_fb->bits_per_pixel) {
@@ -444,7 +444,7 @@ retry:
                  * We don't shutdown the display controller because new buffer
                  * will end up in same spot.
                  */
-                if (!atomic && fb && fb != crtc->fb) {
+                if (!atomic && fb && fb != crtc->primary->fb) {
                         struct radeon_bo *old_rbo;
                         unsigned long nsize, osize;
 
@@ -555,7 +555,7 @@ retry:
         WREG32(RADEON_CRTC_OFFSET + radeon_crtc->crtc_offset, crtc_offset);
         WREG32(RADEON_CRTC_PITCH + radeon_crtc->crtc_offset, crtc_pitch);
 
-        if (!atomic && fb && fb != crtc->fb) {
+        if (!atomic && fb && fb != crtc->primary->fb) {
                 radeon_fb = to_radeon_framebuffer(fb);
                 rbo = gem_to_radeon_bo(radeon_fb->obj);
                 r = radeon_bo_reserve(rbo, false);
@@ -599,7 +599,7 @@ static bool radeon_set_crtc_timing(struct drm_crtc *crtc, struct drm_display_mod
                 }
         }
 
-        switch (crtc->fb->bits_per_pixel) {
+        switch (crtc->primary->fb->bits_per_pixel) {
         case 8:
                 format = 2;
                 break;
@@ -1087,12 +1087,12 @@ static void radeon_crtc_commit(struct drm_crtc *crtc)
 static void radeon_crtc_disable(struct drm_crtc *crtc)
 {
         radeon_crtc_dpms(crtc, DRM_MODE_DPMS_OFF);
-        if (crtc->fb) {
+        if (crtc->primary->fb) {
                 int r;
                 struct radeon_framebuffer *radeon_fb;
                 struct radeon_bo *rbo;
 
-                radeon_fb = to_radeon_framebuffer(crtc->fb);
+                radeon_fb = to_radeon_framebuffer(crtc->primary->fb);
                 rbo = gem_to_radeon_bo(radeon_fb->obj);
                 r = radeon_bo_reserve(rbo, false);
                 if (unlikely(r))
diff --git a/drivers/gpu/drm/radeon/radeon_mode.h b/drivers/gpu/drm/radeon/radeon_mode.h
index 402dbe32c234..832d9fa1a4c4 100644
--- a/drivers/gpu/drm/radeon/radeon_mode.h
+++ b/drivers/gpu/drm/radeon/radeon_mode.h
@@ -192,6 +192,7 @@ struct radeon_i2c_chan {
                 struct i2c_algo_dp_aux_data dp;
         } algo;
         struct radeon_i2c_bus_rec rec;
+        struct drm_dp_aux aux;
 };
 
 /* mostly for macs, but really any system without connector tables */
@@ -690,6 +691,9 @@ extern u8 radeon_dp_getsinktype(struct radeon_connector *radeon_connector);
 extern bool radeon_dp_getdpcd(struct radeon_connector *radeon_connector);
 extern int radeon_dp_get_panel_mode(struct drm_encoder *encoder,
                                     struct drm_connector *connector);
+extern void radeon_dp_set_rx_power_state(struct drm_connector *connector,
+                                         u8 power_state);
+extern void radeon_dp_aux_init(struct radeon_connector *radeon_connector);
 extern void atombios_dig_encoder_setup(struct drm_encoder *encoder, int action, int panel_mode);
 extern void radeon_atom_encoder_init(struct radeon_device *rdev);
 extern void radeon_atom_disp_eng_pll_init(struct radeon_device *rdev);
diff --git a/drivers/gpu/drm/radeon/radeon_object.c b/drivers/gpu/drm/radeon/radeon_object.c
index 08595cf90b01..19bec0dbfa38 100644
--- a/drivers/gpu/drm/radeon/radeon_object.c
+++ b/drivers/gpu/drm/radeon/radeon_object.c
@@ -56,11 +56,36 @@ static void radeon_bo_clear_va(struct radeon_bo *bo)
         }
 }
 
+static void radeon_update_memory_usage(struct radeon_bo *bo,
+                                       unsigned mem_type, int sign)
+{
+        struct radeon_device *rdev = bo->rdev;
+        u64 size = (u64)bo->tbo.num_pages << PAGE_SHIFT;
+
+        switch (mem_type) {
+        case TTM_PL_TT:
+                if (sign > 0)
+                        atomic64_add(size, &rdev->gtt_usage);
+                else
+                        atomic64_sub(size, &rdev->gtt_usage);
+                break;
+        case TTM_PL_VRAM:
+                if (sign > 0)
+                        atomic64_add(size, &rdev->vram_usage);
+                else
+                        atomic64_sub(size, &rdev->vram_usage);
+                break;
+        }
+}
+
 static void radeon_ttm_bo_destroy(struct ttm_buffer_object *tbo)
 {
         struct radeon_bo *bo;
 
         bo = container_of(tbo, struct radeon_bo, tbo);
+
+        radeon_update_memory_usage(bo, bo->tbo.mem.mem_type, -1);
+
         mutex_lock(&bo->rdev->gem.mutex);
         list_del_init(&bo->list);
         mutex_unlock(&bo->rdev->gem.mutex);
@@ -79,7 +104,7 @@ bool radeon_ttm_bo_is_radeon_bo(struct ttm_buffer_object *bo)
 
 void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain)
 {
-        u32 c = 0;
+        u32 c = 0, i;
 
         rbo->placement.fpfn = 0;
         rbo->placement.lpfn = 0;
@@ -106,6 +131,17 @@ void radeon_ttm_placement_from_domain(struct radeon_bo *rbo, u32 domain)
                 rbo->placements[c++] = TTM_PL_MASK_CACHING | TTM_PL_FLAG_SYSTEM;
         rbo->placement.num_placement = c;
         rbo->placement.num_busy_placement = c;
+
+        /*
+         * Use two-ended allocation depending on the buffer size to
+         * improve fragmentation quality.
+         * 512kb was measured as the most optimal number.
+         */
+        if (rbo->tbo.mem.size > 512 * 1024) {
+                for (i = 0; i < c; i++) {
+                        rbo->placements[i] |= TTM_PL_FLAG_TOPDOWN;
+                }
+        }
 }
 
 int radeon_bo_create(struct radeon_device *rdev,
@@ -120,7 +156,6 @@ int radeon_bo_create(struct radeon_device *rdev,
 
         size = ALIGN(size, PAGE_SIZE);
 
-        rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping;
         if (kernel) {
                 type = ttm_bo_type_kernel;
         } else if (sg) {
@@ -145,6 +180,9 @@ int radeon_bo_create(struct radeon_device *rdev,
         bo->surface_reg = -1;
         INIT_LIST_HEAD(&bo->list);
         INIT_LIST_HEAD(&bo->va);
+        bo->initial_domain = domain & (RADEON_GEM_DOMAIN_VRAM |
+                                       RADEON_GEM_DOMAIN_GTT |
+                                       RADEON_GEM_DOMAIN_CPU);
         radeon_ttm_placement_from_domain(bo, domain);
         /* Kernel allocation are uninterruptible */
         down_read(&rdev->pm.mclk_lock);
@@ -338,39 +376,105 @@ void radeon_bo_fini(struct radeon_device *rdev)
         arch_phys_wc_del(rdev->mc.vram_mtrr);
 }
 
-void radeon_bo_list_add_object(struct radeon_bo_list *lobj,
-                                struct list_head *head)
+/* Returns how many bytes TTM can move per IB.
+ */
+static u64 radeon_bo_get_threshold_for_moves(struct radeon_device *rdev)
 {
-        if (lobj->written) {
-                list_add(&lobj->tv.head, head);
-        } else {
-                list_add_tail(&lobj->tv.head, head);
-        }
+        u64 real_vram_size = rdev->mc.real_vram_size;
+        u64 vram_usage = atomic64_read(&rdev->vram_usage);
+
+        /* This function is based on the current VRAM usage.
+         *
+         * - If all of VRAM is free, allow relocating the number of bytes that
+         *   is equal to 1/4 of the size of VRAM for this IB.
+
+         * - If more than one half of VRAM is occupied, only allow relocating
+         *   1 MB of data for this IB.
+         *
+         * - From 0 to one half of used VRAM, the threshold decreases
+         *   linearly.
+         *         __________________
+         * 1/4 of -|\               |
+         * VRAM    | \              |
+         *         |  \             |
+         *         |   \            |
+         *         |    \           |
+         *         |     \          |
+         *         |      \         |
+         *         |       \________|1 MB
+         *         |----------------|
+         *    VRAM 0 %             100 %
+         *         used            used
+         *
+         * Note: It's a threshold, not a limit. The threshold must be crossed
+         * for buffer relocations to stop, so any buffer of an arbitrary size
+         * can be moved as long as the threshold isn't crossed before
+         * the relocation takes place. We don't want to disable buffer
+         * relocations completely.
+         *
+         * The idea is that buffers should be placed in VRAM at creation time
+         * and TTM should only do a minimum number of relocations during
+         * command submission. In practice, you need to submit at least
+         * a dozen IBs to move all buffers to VRAM if they are in GTT.
+         *
+         * Also, things can get pretty crazy under memory pressure and actual
+         * VRAM usage can change a lot, so playing safe even at 50% does
+         * consistently increase performance.
+         */
+
+        u64 half_vram = real_vram_size >> 1;
+        u64 half_free_vram = vram_usage >= half_vram ? 0 : half_vram - vram_usage;
+        u64 bytes_moved_threshold = half_free_vram >> 1;
+        return max(bytes_moved_threshold, 1024*1024ull);
 }
 
-int radeon_bo_list_validate(struct ww_acquire_ctx *ticket,
+int radeon_bo_list_validate(struct radeon_device *rdev,
+                            struct ww_acquire_ctx *ticket,
                             struct list_head *head, int ring)
 {
-        struct radeon_bo_list *lobj;
+        struct radeon_cs_reloc *lobj;
         struct radeon_bo *bo;
-        u32 domain;
         int r;
+        u64 bytes_moved = 0, initial_bytes_moved;
+        u64 bytes_moved_threshold = radeon_bo_get_threshold_for_moves(rdev);
 
         r = ttm_eu_reserve_buffers(ticket, head);
         if (unlikely(r != 0)) {
                 return r;
         }
+
         list_for_each_entry(lobj, head, tv.head) {
-                bo = lobj->bo;
+                bo = lobj->robj;
                 if (!bo->pin_count) {
-                        domain = lobj->domain;
-                        
+                        u32 domain = lobj->domain;
+                        u32 current_domain =
+                                radeon_mem_type_to_domain(bo->tbo.mem.mem_type);
+
+                        /* Check if this buffer will be moved and don't move it
+                         * if we have moved too many buffers for this IB already.
+                         *
+                         * Note that this allows moving at least one buffer of
+                         * any size, because it doesn't take the current "bo"
+                         * into account. We don't want to disallow buffer moves
+                         * completely.
+                         */
+                        if (current_domain != RADEON_GEM_DOMAIN_CPU &&
+                            (domain & current_domain) == 0 && /* will be moved */
+                            bytes_moved > bytes_moved_threshold) {
+                                /* don't move it */
+                                domain = current_domain;
+                        }
+
                 retry:
                         radeon_ttm_placement_from_domain(bo, domain);
                         if (ring == R600_RING_TYPE_UVD_INDEX)
                                 radeon_uvd_force_into_uvd_segment(bo);
-                        r = ttm_bo_validate(&bo->tbo, &bo->placement,
-                                                true, false);
+
+                        initial_bytes_moved = atomic64_read(&rdev->num_bytes_moved);
+                        r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
+                        bytes_moved += atomic64_read(&rdev->num_bytes_moved) -
+                                       initial_bytes_moved;
+
                         if (unlikely(r)) {
                                 if (r != -ERESTARTSYS && domain != lobj->alt_domain) {
                                         domain = lobj->alt_domain;
@@ -564,14 +668,23 @@ int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved,
 }
 
 void radeon_bo_move_notify(struct ttm_buffer_object *bo,
-                           struct ttm_mem_reg *mem)
+                           struct ttm_mem_reg *new_mem)
 {
         struct radeon_bo *rbo;
+
         if (!radeon_ttm_bo_is_radeon_bo(bo))
                 return;
+
         rbo = container_of(bo, struct radeon_bo, tbo);
         radeon_bo_check_tiling(rbo, 0, 1);
         radeon_vm_bo_invalidate(rbo->rdev, rbo);
+
+        /* update statistics */
+        if (!new_mem)
+                return;
+
+        radeon_update_memory_usage(rbo, bo->mem.mem_type, -1);
+        radeon_update_memory_usage(rbo, new_mem->mem_type, 1);
 }
 
 int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
diff --git a/drivers/gpu/drm/radeon/radeon_object.h b/drivers/gpu/drm/radeon/radeon_object.h
index 209b11150263..9e7b25a0629d 100644
--- a/drivers/gpu/drm/radeon/radeon_object.h
+++ b/drivers/gpu/drm/radeon/radeon_object.h
@@ -138,9 +138,8 @@ extern int radeon_bo_evict_vram(struct radeon_device *rdev);
 extern void radeon_bo_force_delete(struct radeon_device *rdev);
 extern int radeon_bo_init(struct radeon_device *rdev);
 extern void radeon_bo_fini(struct radeon_device *rdev);
-extern void radeon_bo_list_add_object(struct radeon_bo_list *lobj,
-                                struct list_head *head);
-extern int radeon_bo_list_validate(struct ww_acquire_ctx *ticket,
+extern int radeon_bo_list_validate(struct radeon_device *rdev,
+                                   struct ww_acquire_ctx *ticket,
                                    struct list_head *head, int ring);
 extern int radeon_bo_fbdev_mmap(struct radeon_bo *bo,
                                 struct vm_area_struct *vma);
@@ -151,7 +150,7 @@ extern void radeon_bo_get_tiling_flags(struct radeon_bo *bo,
 extern int radeon_bo_check_tiling(struct radeon_bo *bo, bool has_moved,
                                 bool force_drop);
 extern void radeon_bo_move_notify(struct ttm_buffer_object *bo,
-                                        struct ttm_mem_reg *mem);
+                                  struct ttm_mem_reg *new_mem);
 extern int radeon_bo_fault_reserve_notify(struct ttm_buffer_object *bo);
 extern int radeon_bo_get_surface_reg(struct radeon_bo *bo);
 
@@ -181,7 +180,7 @@ extern int radeon_sa_bo_manager_suspend(struct radeon_device *rdev,
 extern int radeon_sa_bo_new(struct radeon_device *rdev,
                             struct radeon_sa_manager *sa_manager,
                             struct radeon_sa_bo **sa_bo,
-                            unsigned size, unsigned align, bool block);
+                            unsigned size, unsigned align);
 extern void radeon_sa_bo_free(struct radeon_device *rdev,
                               struct radeon_sa_bo **sa_bo,
                               struct radeon_fence *fence);
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index 8e8153e471c2..ee738a524639 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -260,7 +260,7 @@ static void radeon_pm_set_clocks(struct radeon_device *rdev)
                 if (!ring->ready) {
                         continue;
                 }
-                r = radeon_fence_wait_empty_locked(rdev, i);
+                r = radeon_fence_wait_empty(rdev, i);
                 if (r) {
                         /* needs a GPU reset dont reset here */
                         mutex_unlock(&rdev->ring_lock);
@@ -826,6 +826,9 @@ static void radeon_dpm_change_power_state_locked(struct radeon_device *rdev)
 
         /* no need to reprogram if nothing changed unless we are on BTC+ */
         if (rdev->pm.dpm.current_ps == rdev->pm.dpm.requested_ps) {
+                /* vce just modifies an existing state so force a change */
+                if (ps->vce_active != rdev->pm.dpm.vce_active)
+                        goto force;
                 if ((rdev->family < CHIP_BARTS) || (rdev->flags & RADEON_IS_IGP)) {
                         /* for pre-BTC and APUs if the num crtcs changed but state is the same,
                          * all we need to do is update the display configuration.
@@ -862,16 +865,21 @@ static void radeon_dpm_change_power_state_locked(struct radeon_device *rdev)
                 }
         }
 
+force:
         if (radeon_dpm == 1) {
                 printk("switching from power state:\n");
                 radeon_dpm_print_power_state(rdev, rdev->pm.dpm.current_ps);
                 printk("switching to power state:\n");
                 radeon_dpm_print_power_state(rdev, rdev->pm.dpm.requested_ps);
         }
+
         mutex_lock(&rdev->ddev->struct_mutex);
         down_write(&rdev->pm.mclk_lock);
         mutex_lock(&rdev->ring_lock);
 
+        /* update whether vce is active */
+        ps->vce_active = rdev->pm.dpm.vce_active;
+
         ret = radeon_dpm_pre_set_power_state(rdev);
         if (ret)
                 goto done;
@@ -888,7 +896,7 @@ static void radeon_dpm_change_power_state_locked(struct radeon_device *rdev)
         for (i = 0; i < RADEON_NUM_RINGS; i++) {
                 struct radeon_ring *ring = &rdev->ring[i];
                 if (ring->ready)
-                        radeon_fence_wait_empty_locked(rdev, i);
+                        radeon_fence_wait_empty(rdev, i);
         }
 
         /* program the new power state */
@@ -935,8 +943,6 @@ void radeon_dpm_enable_uvd(struct radeon_device *rdev, bool enable)
                 if (enable) {
                         mutex_lock(&rdev->pm.mutex);
                         rdev->pm.dpm.uvd_active = true;
-                        /* disable this for now */
-#if 0
                         if ((rdev->pm.dpm.sd == 1) && (rdev->pm.dpm.hd == 0))
                                 dpm_state = POWER_STATE_TYPE_INTERNAL_UVD_SD;
                         else if ((rdev->pm.dpm.sd == 2) && (rdev->pm.dpm.hd == 0))
@@ -946,7 +952,6 @@ void radeon_dpm_enable_uvd(struct radeon_device *rdev, bool enable)
                         else if ((rdev->pm.dpm.sd == 0) && (rdev->pm.dpm.hd == 2))
                                 dpm_state = POWER_STATE_TYPE_INTERNAL_UVD_HD2;
                         else
-#endif
                                 dpm_state = POWER_STATE_TYPE_INTERNAL_UVD;
                         rdev->pm.dpm.state = dpm_state;
                         mutex_unlock(&rdev->pm.mutex);
@@ -960,6 +965,23 @@ void radeon_dpm_enable_uvd(struct radeon_device *rdev, bool enable)
         }
 }
 
+void radeon_dpm_enable_vce(struct radeon_device *rdev, bool enable)
+{
+        if (enable) {
+                mutex_lock(&rdev->pm.mutex);
+                rdev->pm.dpm.vce_active = true;
+                /* XXX select vce level based on ring/task */
+                rdev->pm.dpm.vce_level = RADEON_VCE_LEVEL_AC_ALL;
+                mutex_unlock(&rdev->pm.mutex);
+        } else {
+                mutex_lock(&rdev->pm.mutex);
+                rdev->pm.dpm.vce_active = false;
+                mutex_unlock(&rdev->pm.mutex);
+        }
+
+        radeon_pm_compute_clocks(rdev);
+}
+
 static void radeon_pm_suspend_old(struct radeon_device *rdev)
 {
         mutex_lock(&rdev->pm.mutex);
diff --git a/drivers/gpu/drm/radeon/radeon_ring.c b/drivers/gpu/drm/radeon/radeon_ring.c
index 15e44a7281ab..f8050f5429e2 100644
--- a/drivers/gpu/drm/radeon/radeon_ring.c
+++ b/drivers/gpu/drm/radeon/radeon_ring.c
@@ -63,7 +63,7 @@ int radeon_ib_get(struct radeon_device *rdev, int ring,
 {
         int r;
 
-        r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &ib->sa_bo, size, 256, true);
+        r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &ib->sa_bo, size, 256);
         if (r) {
                 dev_err(rdev->dev, "failed to get a new IB (%d)\n", r);
                 return r;
@@ -145,6 +145,13 @@ int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib,
                 return r;
         }
 
+        /* grab a vm id if necessary */
+        if (ib->vm) {
+                struct radeon_fence *vm_id_fence;
+                vm_id_fence = radeon_vm_grab_id(rdev, ib->vm, ib->ring);
+                radeon_semaphore_sync_to(ib->semaphore, vm_id_fence);
+        }
+
         /* sync with other rings */
         r = radeon_semaphore_sync_rings(rdev, ib->semaphore, ib->ring);
         if (r) {
@@ -153,11 +160,9 @@ int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib,
                 return r;
         }
 
-        /* if we can't remember our last VM flush then flush now! */
-        /* XXX figure out why we have to flush for every IB */
-        if (ib->vm /*&& !ib->vm->last_flush*/) {
-                radeon_ring_vm_flush(rdev, ib->ring, ib->vm);
-        }
+        if (ib->vm)
+                radeon_vm_flush(rdev, ib->vm, ib->ring);
+
         if (const_ib) {
                 radeon_ring_ib_execute(rdev, const_ib->ring, const_ib);
                 radeon_semaphore_free(rdev, &const_ib->semaphore, NULL);
@@ -172,10 +177,10 @@ int radeon_ib_schedule(struct radeon_device *rdev, struct radeon_ib *ib,
         if (const_ib) {
                 const_ib->fence = radeon_fence_ref(ib->fence);
         }
-        /* we just flushed the VM, remember that */
-        if (ib->vm && !ib->vm->last_flush) {
-                ib->vm->last_flush = radeon_fence_ref(ib->fence);
-        }
+
+        if (ib->vm)
+                radeon_vm_fence(rdev, ib->vm, ib->fence);
+
         radeon_ring_unlock_commit(rdev, ring);
         return 0;
 }
@@ -257,6 +262,7 @@ int radeon_ib_ring_tests(struct radeon_device *rdev)
                 r = radeon_ib_test(rdev, i, ring);
                 if (r) {
                         ring->ready = false;
+                        rdev->needs_reset = false;
 
                         if (i == RADEON_RING_TYPE_GFX_INDEX) {
                                 /* oh, oh, that's really bad */
@@ -342,13 +348,17 @@ bool radeon_ring_supports_scratch_reg(struct radeon_device *rdev,
  */
 void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *ring)
 {
-        ring->rptr = radeon_ring_get_rptr(rdev, ring);
+        uint32_t rptr = radeon_ring_get_rptr(rdev, ring);
+
         /* This works because ring_size is a power of 2 */
-        ring->ring_free_dw = (ring->rptr + (ring->ring_size / 4));
+        ring->ring_free_dw = rptr + (ring->ring_size / 4);
         ring->ring_free_dw -= ring->wptr;
         ring->ring_free_dw &= ring->ptr_mask;
         if (!ring->ring_free_dw) {
+                /* this is an empty ring */
                 ring->ring_free_dw = ring->ring_size / 4;
+                /*  update lockup info to avoid false positive */
+                radeon_ring_lockup_update(rdev, ring);
         }
 }
 
@@ -372,19 +382,13 @@ int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsi
         /* Align requested size with padding so unlock_commit can
          * pad safely */
         radeon_ring_free_size(rdev, ring);
-        if (ring->ring_free_dw == (ring->ring_size / 4)) {
-                /* This is an empty ring update lockup info to avoid
-                 * false positive.
-                 */
-                radeon_ring_lockup_update(ring);
-        }
         ndw = (ndw + ring->align_mask) & ~ring->align_mask;
         while (ndw > (ring->ring_free_dw - 1)) {
                 radeon_ring_free_size(rdev, ring);
                 if (ndw < ring->ring_free_dw) {
                         break;
                 }
-                r = radeon_fence_wait_next_locked(rdev, ring->idx);
+                r = radeon_fence_wait_next(rdev, ring->idx);
                 if (r)
                         return r;
         }
@@ -478,39 +482,17 @@ void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *rin
 }
 
 /**
- * radeon_ring_force_activity - add some nop packets to the ring
- *
- * @rdev: radeon_device pointer
- * @ring: radeon_ring structure holding ring information
- *
- * Add some nop packets to the ring to force activity (all asics).
- * Used for lockup detection to see if the rptr is advancing.
- */
-void radeon_ring_force_activity(struct radeon_device *rdev, struct radeon_ring *ring)
-{
-        int r;
-
-        radeon_ring_free_size(rdev, ring);
-        if (ring->rptr == ring->wptr) {
-                r = radeon_ring_alloc(rdev, ring, 1);
-                if (!r) {
-                        radeon_ring_write(ring, ring->nop);
-                        radeon_ring_commit(rdev, ring);
-                }
-        }
-}
-
-/**
  * radeon_ring_lockup_update - update lockup variables
  *
  * @ring: radeon_ring structure holding ring information
  *
  * Update the last rptr value and timestamp (all asics).
  */
-void radeon_ring_lockup_update(struct radeon_ring *ring)
+void radeon_ring_lockup_update(struct radeon_device *rdev,
+                               struct radeon_ring *ring)
 {
-        ring->last_rptr = ring->rptr;
-        ring->last_activity = jiffies;
+        atomic_set(&ring->last_rptr, radeon_ring_get_rptr(rdev, ring));
+        atomic64_set(&ring->last_activity, jiffies_64);
 }
 
 /**
@@ -518,40 +500,23 @@ void radeon_ring_lockup_update(struct radeon_ring *ring)
  * @rdev:       radeon device structure
  * @ring:       radeon_ring structure holding ring information
  *
- * We don't need to initialize the lockup tracking information as we will either
- * have CP rptr to a different value of jiffies wrap around which will force
- * initialization of the lockup tracking informations.
- *
- * A possible false positivie is if we get call after while and last_cp_rptr ==
- * the current CP rptr, even if it's unlikely it might happen. To avoid this
- * if the elapsed time since last call is bigger than 2 second than we return
- * false and update the tracking information. Due to this the caller must call
- * radeon_ring_test_lockup several time in less than 2sec for lockup to be reported
- * the fencing code should be cautious about that.
- *
- * Caller should write to the ring to force CP to do something so we don't get
- * false positive when CP is just gived nothing to do.
- *
- **/
+ */
 bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
 {
-        unsigned long cjiffies, elapsed;
+        uint32_t rptr = radeon_ring_get_rptr(rdev, ring);
+        uint64_t last = atomic64_read(&ring->last_activity);
+        uint64_t elapsed;
 
-        cjiffies = jiffies;
-        if (!time_after(cjiffies, ring->last_activity)) {
-                /* likely a wrap around */
-                radeon_ring_lockup_update(ring);
+        if (rptr != atomic_read(&ring->last_rptr)) {
+                /* ring is still working, no lockup */
+                radeon_ring_lockup_update(rdev, ring);
                 return false;
         }
-        ring->rptr = radeon_ring_get_rptr(rdev, ring);
-        if (ring->rptr != ring->last_rptr) {
-                /* CP is still working no lockup */
-                radeon_ring_lockup_update(ring);
-                return false;
-        }
-        elapsed = jiffies_to_msecs(cjiffies - ring->last_activity);
+
+        elapsed = jiffies_to_msecs(jiffies_64 - last);
         if (radeon_lockup_timeout && elapsed >= radeon_lockup_timeout) {
-                dev_err(rdev->dev, "GPU lockup CP stall for more than %lumsec\n", elapsed);
+                dev_err(rdev->dev, "ring %d stalled for more than %llumsec\n",
+                        ring->idx, elapsed);
                 return true;
         }
         /* give a chance to the GPU ... */
@@ -709,7 +674,7 @@ int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsig
         if (radeon_debugfs_ring_init(rdev, ring)) {
                 DRM_ERROR("Failed to register debugfs file for rings !\n");
         }
-        radeon_ring_lockup_update(ring);
+        radeon_ring_lockup_update(rdev, ring);
         return 0;
 }
 
@@ -780,8 +745,6 @@ static int radeon_debugfs_ring_info(struct seq_file *m, void *data)
 
         seq_printf(m, "driver's copy of the wptr: 0x%08x [%5d]\n",
                    ring->wptr, ring->wptr);
-        seq_printf(m, "driver's copy of the rptr: 0x%08x [%5d]\n",
-                   ring->rptr, ring->rptr);
         seq_printf(m, "last semaphore signal addr : 0x%016llx\n",
                    ring->last_semaphore_signal_addr);
         seq_printf(m, "last semaphore wait addr   : 0x%016llx\n",
@@ -814,6 +777,8 @@ static int cayman_cp2_index = CAYMAN_RING_TYPE_CP2_INDEX;
 static int radeon_dma1_index = R600_RING_TYPE_DMA_INDEX;
 static int radeon_dma2_index = CAYMAN_RING_TYPE_DMA1_INDEX;
 static int r600_uvd_index = R600_RING_TYPE_UVD_INDEX;
+static int si_vce1_index = TN_RING_TYPE_VCE1_INDEX;
+static int si_vce2_index = TN_RING_TYPE_VCE2_INDEX;
 
 static struct drm_info_list radeon_debugfs_ring_info_list[] = {
         {"radeon_ring_gfx", radeon_debugfs_ring_info, 0, &radeon_gfx_index},
@@ -822,6 +787,8 @@ static struct drm_info_list radeon_debugfs_ring_info_list[] = {
         {"radeon_ring_dma1", radeon_debugfs_ring_info, 0, &radeon_dma1_index},
         {"radeon_ring_dma2", radeon_debugfs_ring_info, 0, &radeon_dma2_index},
         {"radeon_ring_uvd", radeon_debugfs_ring_info, 0, &r600_uvd_index},
+        {"radeon_ring_vce1", radeon_debugfs_ring_info, 0, &si_vce1_index},
+        {"radeon_ring_vce2", radeon_debugfs_ring_info, 0, &si_vce2_index},
 };
 
 static int radeon_debugfs_sa_info(struct seq_file *m, void *data)
diff --git a/drivers/gpu/drm/radeon/radeon_sa.c b/drivers/gpu/drm/radeon/radeon_sa.c
index c0625805cdd7..adcf3e2f07da 100644
--- a/drivers/gpu/drm/radeon/radeon_sa.c
+++ b/drivers/gpu/drm/radeon/radeon_sa.c
@@ -312,7 +312,7 @@ static bool radeon_sa_bo_next_hole(struct radeon_sa_manager *sa_manager,
 int radeon_sa_bo_new(struct radeon_device *rdev,
                      struct radeon_sa_manager *sa_manager,
                      struct radeon_sa_bo **sa_bo,
-                     unsigned size, unsigned align, bool block)
+                     unsigned size, unsigned align)
 {
         struct radeon_fence *fences[RADEON_NUM_RINGS];
         unsigned tries[RADEON_NUM_RINGS];
@@ -353,14 +353,11 @@ int radeon_sa_bo_new(struct radeon_device *rdev,
                 r = radeon_fence_wait_any(rdev, fences, false);
                 spin_lock(&sa_manager->wq.lock);
                 /* if we have nothing to wait for block */
-                if (r == -ENOENT && block) {
+                if (r == -ENOENT) {
                         r = wait_event_interruptible_locked(
                                 sa_manager->wq, 
                                 radeon_sa_event(sa_manager, size, align)
                         );
-
-                } else if (r == -ENOENT) {
-                        r = -ENOMEM;
                 }
 
         } while (!r);
diff --git a/drivers/gpu/drm/radeon/radeon_semaphore.c b/drivers/gpu/drm/radeon/radeon_semaphore.c
index 9006b32d5eed..dbd6bcde92de 100644
--- a/drivers/gpu/drm/radeon/radeon_semaphore.c
+++ b/drivers/gpu/drm/radeon/radeon_semaphore.c
@@ -42,7 +42,7 @@ int radeon_semaphore_create(struct radeon_device *rdev,
                 return -ENOMEM;
         }
         r = radeon_sa_bo_new(rdev, &rdev->ring_tmp_bo, &(*semaphore)->sa_bo,
-                             8 * RADEON_NUM_SYNCS, 8, true);
+                             8 * RADEON_NUM_SYNCS, 8);
         if (r) {
                 kfree(*semaphore);
                 *semaphore = NULL;
@@ -147,7 +147,9 @@ int radeon_semaphore_sync_rings(struct radeon_device *rdev,
 
                 if (++count > RADEON_NUM_SYNCS) {
                         /* not enough room, wait manually */
-                        radeon_fence_wait_locked(fence);
+                        r = radeon_fence_wait(fence, false);
+                        if (r)
+                                return r;
                         continue;
                 }
 
@@ -161,7 +163,9 @@ int radeon_semaphore_sync_rings(struct radeon_device *rdev,
                 if (!radeon_semaphore_emit_signal(rdev, i, semaphore)) {
                         /* signaling wasn't successful wait manually */
                         radeon_ring_undo(&rdev->ring[i]);
-                        radeon_fence_wait_locked(fence);
+                        r = radeon_fence_wait(fence, false);
+                        if (r)
+                                return r;
                         continue;
                 }
 
@@ -169,7 +173,9 @@ int radeon_semaphore_sync_rings(struct radeon_device *rdev,
                 if (!radeon_semaphore_emit_wait(rdev, ring, semaphore)) {
                         /* waiting wasn't successful wait manually */
                         radeon_ring_undo(&rdev->ring[i]);
-                        radeon_fence_wait_locked(fence);
+                        r = radeon_fence_wait(fence, false);
+                        if (r)
+                                return r;
                         continue;
                 }
 
diff --git a/drivers/gpu/drm/radeon/radeon_test.c b/drivers/gpu/drm/radeon/radeon_test.c
index 12e8099a0823..3a13e0d1055c 100644
--- a/drivers/gpu/drm/radeon/radeon_test.c
+++ b/drivers/gpu/drm/radeon/radeon_test.c
@@ -257,20 +257,36 @@ static int radeon_test_create_and_emit_fence(struct radeon_device *rdev,
                                              struct radeon_ring *ring,
                                              struct radeon_fence **fence)
 {
+        uint32_t handle = ring->idx ^ 0xdeafbeef;
         int r;
 
         if (ring->idx == R600_RING_TYPE_UVD_INDEX) {
-                r = radeon_uvd_get_create_msg(rdev, ring->idx, 1, NULL);
+                r = radeon_uvd_get_create_msg(rdev, ring->idx, handle, NULL);
                 if (r) {
                         DRM_ERROR("Failed to get dummy create msg\n");
                         return r;
                 }
 
-                r = radeon_uvd_get_destroy_msg(rdev, ring->idx, 1, fence);
+                r = radeon_uvd_get_destroy_msg(rdev, ring->idx, handle, fence);
                 if (r) {
                         DRM_ERROR("Failed to get dummy destroy msg\n");
                         return r;
                 }
+
+        } else if (ring->idx == TN_RING_TYPE_VCE1_INDEX ||
+                   ring->idx == TN_RING_TYPE_VCE2_INDEX) {
+                r = radeon_vce_get_create_msg(rdev, ring->idx, handle, NULL);
+                if (r) {
+                        DRM_ERROR("Failed to get dummy create msg\n");
+                        return r;
+                }
+
+                r = radeon_vce_get_destroy_msg(rdev, ring->idx, handle, fence);
+                if (r) {
+                        DRM_ERROR("Failed to get dummy destroy msg\n");
+                        return r;
+                }
+
         } else {
                 r = radeon_ring_lock(rdev, ring, 64);
                 if (r) {
@@ -486,6 +502,16 @@ out_cleanup:
                 printk(KERN_WARNING "Error while testing ring sync (%d).\n", r);
 }
 
+static bool radeon_test_sync_possible(struct radeon_ring *ringA,
+                                      struct radeon_ring *ringB)
+{
+        if (ringA->idx == TN_RING_TYPE_VCE2_INDEX &&
+            ringB->idx == TN_RING_TYPE_VCE1_INDEX)
+                return false;
+
+        return true;
+}
+
 void radeon_test_syncing(struct radeon_device *rdev)
 {
         int i, j, k;
@@ -500,6 +526,9 @@ void radeon_test_syncing(struct radeon_device *rdev)
                         if (!ringB->ready)
                                 continue;
 
+                        if (!radeon_test_sync_possible(ringA, ringB))
+                                continue;
+
                         DRM_INFO("Testing syncing between rings %d and %d...\n", i, j);
                         radeon_test_ring_sync(rdev, ringA, ringB);
 
@@ -511,6 +540,12 @@ void radeon_test_syncing(struct radeon_device *rdev)
                                 if (!ringC->ready)
                                         continue;
 
+                                if (!radeon_test_sync_possible(ringA, ringC))
+                                        continue;
+
+                                if (!radeon_test_sync_possible(ringB, ringC))
+                                        continue;
+
                                 DRM_INFO("Testing syncing between rings %d, %d and %d...\n", i, j, k);
                                 radeon_test_ring_sync2(rdev, ringA, ringB, ringC);
 
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 040a2a10ea17..c8a8a5144ec1 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -406,8 +406,14 @@ static int radeon_bo_move(struct ttm_buffer_object *bo,
         if (r) {
 memcpy:
                 r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
+                if (r) {
+                        return r;
+                }
         }
-        return r;
+
+        /* update statistics */
+        atomic64_add((u64)bo->num_pages << PAGE_SHIFT, &rdev->num_bytes_moved);
+        return 0;
 }
 
 static int radeon_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem)
@@ -701,7 +707,9 @@ int radeon_ttm_init(struct radeon_device *rdev)
         /* No others user of address space so set it to 0 */
         r = ttm_bo_device_init(&rdev->mman.bdev,
                                rdev->mman.bo_global_ref.ref.object,
-                               &radeon_bo_driver, DRM_FILE_PAGE_OFFSET,
+                               &radeon_bo_driver,
+                               rdev->ddev->anon_inode->i_mapping,
+                               DRM_FILE_PAGE_OFFSET,
                                rdev->need_dma32);
         if (r) {
                 DRM_ERROR("failed initializing buffer object driver(%d).\n", r);
@@ -742,7 +750,6 @@ int radeon_ttm_init(struct radeon_device *rdev)
         }
         DRM_INFO("radeon: %uM of GTT memory ready.\n",
                  (unsigned)(rdev->mc.gtt_size / (1024 * 1024)));
-        rdev->mman.bdev.dev_mapping = rdev->ddev->dev_mapping;
 
         r = radeon_ttm_debugfs_init(rdev);
         if (r) {
diff --git a/drivers/gpu/drm/radeon/radeon_uvd.c b/drivers/gpu/drm/radeon/radeon_uvd.c
index 3e6804b2b2ef..5748bdaeacce 100644
--- a/drivers/gpu/drm/radeon/radeon_uvd.c
+++ b/drivers/gpu/drm/radeon/radeon_uvd.c
@@ -455,7 +455,7 @@ static int radeon_uvd_cs_reloc(struct radeon_cs_parser *p,
         }
 
         reloc = p->relocs_ptr[(idx / 4)];
-        start = reloc->lobj.gpu_offset;
+        start = reloc->gpu_offset;
         end = start + radeon_bo_size(reloc->robj);
         start += offset;
 
@@ -807,8 +807,7 @@ void radeon_uvd_note_usage(struct radeon_device *rdev)
                     (rdev->pm.dpm.hd != hd)) {
                         rdev->pm.dpm.sd = sd;
                         rdev->pm.dpm.hd = hd;
-                        /* disable this for now */
-                        /*streams_changed = true;*/
+                        streams_changed = true;
                 }
         }
 
diff --git a/drivers/gpu/drm/radeon/radeon_vce.c b/drivers/gpu/drm/radeon/radeon_vce.c
new file mode 100644
index 000000000000..76e9904bc537
--- /dev/null
+++ b/drivers/gpu/drm/radeon/radeon_vce.c
@@ -0,0 +1,699 @@
+/*
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * Authors: Christian König <christian.koenig@amd.com>
+ */
+
+#include <linux/firmware.h>
+#include <linux/module.h>
+#include <drm/drmP.h>
+#include <drm/drm.h>
+
+#include "radeon.h"
+#include "radeon_asic.h"
+#include "sid.h"
+
+/* 1 second timeout */
+#define VCE_IDLE_TIMEOUT_MS     1000
+
+/* Firmware Names */
+#define FIRMWARE_BONAIRE        "radeon/BONAIRE_vce.bin"
+
+MODULE_FIRMWARE(FIRMWARE_BONAIRE);
+
+static void radeon_vce_idle_work_handler(struct work_struct *work);
+
+/**
+ * radeon_vce_init - allocate memory, load vce firmware
+ *
+ * @rdev: radeon_device pointer
+ *
+ * First step to get VCE online, allocate memory and load the firmware
+ */
+int radeon_vce_init(struct radeon_device *rdev)
+{
+        static const char *fw_version = "[ATI LIB=VCEFW,";
+        static const char *fb_version = "[ATI LIB=VCEFWSTATS,";
+        unsigned long size;
+        const char *fw_name, *c;
+        uint8_t start, mid, end;
+        int i, r;
+
+        INIT_DELAYED_WORK(&rdev->vce.idle_work, radeon_vce_idle_work_handler);
+
+        switch (rdev->family) {
+        case CHIP_BONAIRE:
+        case CHIP_KAVERI:
+        case CHIP_KABINI:
+                fw_name = FIRMWARE_BONAIRE;
+                break;
+
+        default:
+                return -EINVAL;
+        }
+
+        r = request_firmware(&rdev->vce_fw, fw_name, rdev->dev);
+        if (r) {
+                dev_err(rdev->dev, "radeon_vce: Can't load firmware \"%s\"\n",
+                        fw_name);
+                return r;
+        }
+
+        /* search for firmware version */
+
+        size = rdev->vce_fw->size - strlen(fw_version) - 9;
+        c = rdev->vce_fw->data;
+        for (;size > 0; --size, ++c)
+                if (strncmp(c, fw_version, strlen(fw_version)) == 0)
+                        break;
+
+        if (size == 0)
+                return -EINVAL;
+
+        c += strlen(fw_version);
+        if (sscanf(c, "%2hhd.%2hhd.%2hhd]", &start, &mid, &end) != 3)
+                return -EINVAL;
+
+        /* search for feedback version */
+
+        size = rdev->vce_fw->size - strlen(fb_version) - 3;
+        c = rdev->vce_fw->data;
+        for (;size > 0; --size, ++c)
+                if (strncmp(c, fb_version, strlen(fb_version)) == 0)
+                        break;
+
+        if (size == 0)
+                return -EINVAL;
+
+        c += strlen(fb_version);
+        if (sscanf(c, "%2u]", &rdev->vce.fb_version) != 1)
+                return -EINVAL;
+
+        DRM_INFO("Found VCE firmware/feedback version %hhd.%hhd.%hhd / %d!\n",
+                 start, mid, end, rdev->vce.fb_version);
+
+        rdev->vce.fw_version = (start << 24) | (mid << 16) | (end << 8);
+
+        /* we can only work with this fw version for now */
+        if (rdev->vce.fw_version != ((40 << 24) | (2 << 16) | (2 << 8)))
+                return -EINVAL;
+
+        /* allocate firmware, stack and heap BO */
+
+        size = RADEON_GPU_PAGE_ALIGN(rdev->vce_fw->size) +
+               RADEON_VCE_STACK_SIZE + RADEON_VCE_HEAP_SIZE;
+        r = radeon_bo_create(rdev, size, PAGE_SIZE, true,
+                             RADEON_GEM_DOMAIN_VRAM, NULL, &rdev->vce.vcpu_bo);
+        if (r) {
+                dev_err(rdev->dev, "(%d) failed to allocate VCE bo\n", r);
+                return r;
+        }
+
+        r = radeon_bo_reserve(rdev->vce.vcpu_bo, false);
+        if (r) {
+                radeon_bo_unref(&rdev->vce.vcpu_bo);
+                dev_err(rdev->dev, "(%d) failed to reserve VCE bo\n", r);
+                return r;
+        }
+
+        r = radeon_bo_pin(rdev->vce.vcpu_bo, RADEON_GEM_DOMAIN_VRAM,
+                          &rdev->vce.gpu_addr);
+        radeon_bo_unreserve(rdev->vce.vcpu_bo);
+        if (r) {
+                radeon_bo_unref(&rdev->vce.vcpu_bo);
+                dev_err(rdev->dev, "(%d) VCE bo pin failed\n", r);
+                return r;
+        }
+
+        for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) {
+                atomic_set(&rdev->vce.handles[i], 0);
+                rdev->vce.filp[i] = NULL;
+        }
+
+        return 0;
+}
+
+/**
+ * radeon_vce_fini - free memory
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Last step on VCE teardown, free firmware memory
+ */
+void radeon_vce_fini(struct radeon_device *rdev)
+{
+        if (rdev->vce.vcpu_bo == NULL)
+                return;
+
+        radeon_bo_unref(&rdev->vce.vcpu_bo);
+
+        release_firmware(rdev->vce_fw);
+}
+
+/**
+ * radeon_vce_suspend - unpin VCE fw memory
+ *
+ * @rdev: radeon_device pointer
+ *
+ */
+int radeon_vce_suspend(struct radeon_device *rdev)
+{
+        int i;
+
+        if (rdev->vce.vcpu_bo == NULL)
+                return 0;
+
+        for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i)
+                if (atomic_read(&rdev->vce.handles[i]))
+                        break;
+
+        if (i == RADEON_MAX_VCE_HANDLES)
+                return 0;
+
+        /* TODO: suspending running encoding sessions isn't supported */
+        return -EINVAL;
+}
+
+/**
+ * radeon_vce_resume - pin VCE fw memory
+ *
+ * @rdev: radeon_device pointer
+ *
+ */
+int radeon_vce_resume(struct radeon_device *rdev)
+{
+        void *cpu_addr;
+        int r;
+
+        if (rdev->vce.vcpu_bo == NULL)
+                return -EINVAL;
+
+        r = radeon_bo_reserve(rdev->vce.vcpu_bo, false);
+        if (r) {
+                dev_err(rdev->dev, "(%d) failed to reserve VCE bo\n", r);
+                return r;
+        }
+
+        r = radeon_bo_kmap(rdev->vce.vcpu_bo, &cpu_addr);
+        if (r) {
+                radeon_bo_unreserve(rdev->vce.vcpu_bo);
+                dev_err(rdev->dev, "(%d) VCE map failed\n", r);
+                return r;
+        }
+
+        memcpy(cpu_addr, rdev->vce_fw->data, rdev->vce_fw->size);
+
+        radeon_bo_kunmap(rdev->vce.vcpu_bo);
+
+        radeon_bo_unreserve(rdev->vce.vcpu_bo);
+
+        return 0;
+}
+
+/**
+ * radeon_vce_idle_work_handler - power off VCE
+ *
+ * @work: pointer to work structure
+ *
+ * power of VCE when it's not used any more
+ */
+static void radeon_vce_idle_work_handler(struct work_struct *work)
+{
+        struct radeon_device *rdev =
+                container_of(work, struct radeon_device, vce.idle_work.work);
+
+        if ((radeon_fence_count_emitted(rdev, TN_RING_TYPE_VCE1_INDEX) == 0) &&
+            (radeon_fence_count_emitted(rdev, TN_RING_TYPE_VCE2_INDEX) == 0)) {
+                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
+                        radeon_dpm_enable_vce(rdev, false);
+                } else {
+                        radeon_set_vce_clocks(rdev, 0, 0);
+                }
+        } else {
+                schedule_delayed_work(&rdev->vce.idle_work,
+                                      msecs_to_jiffies(VCE_IDLE_TIMEOUT_MS));
+        }
+}
+
+/**
+ * radeon_vce_note_usage - power up VCE
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Make sure VCE is powerd up when we want to use it
+ */
+void radeon_vce_note_usage(struct radeon_device *rdev)
+{
+        bool streams_changed = false;
+        bool set_clocks = !cancel_delayed_work_sync(&rdev->vce.idle_work);
+        set_clocks &= schedule_delayed_work(&rdev->vce.idle_work,
+                                            msecs_to_jiffies(VCE_IDLE_TIMEOUT_MS));
+
+        if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
+                /* XXX figure out if the streams changed */
+                streams_changed = false;
+        }
+
+        if (set_clocks || streams_changed) {
+                if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
+                        radeon_dpm_enable_vce(rdev, true);
+                } else {
+                        radeon_set_vce_clocks(rdev, 53300, 40000);
+                }
+        }
+}
+
+/**
+ * radeon_vce_free_handles - free still open VCE handles
+ *
+ * @rdev: radeon_device pointer
+ * @filp: drm file pointer
+ *
+ * Close all VCE handles still open by this file pointer
+ */
+void radeon_vce_free_handles(struct radeon_device *rdev, struct drm_file *filp)
+{
+        int i, r;
+        for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) {
+                uint32_t handle = atomic_read(&rdev->vce.handles[i]);
+                if (!handle || rdev->vce.filp[i] != filp)
+                        continue;
+
+                radeon_vce_note_usage(rdev);
+
+                r = radeon_vce_get_destroy_msg(rdev, TN_RING_TYPE_VCE1_INDEX,
+                                               handle, NULL);
+                if (r)
+                        DRM_ERROR("Error destroying VCE handle (%d)!\n", r);
+
+                rdev->vce.filp[i] = NULL;
+                atomic_set(&rdev->vce.handles[i], 0);
+        }
+}
+
+/**
+ * radeon_vce_get_create_msg - generate a VCE create msg
+ *
+ * @rdev: radeon_device pointer
+ * @ring: ring we should submit the msg to
+ * @handle: VCE session handle to use
+ * @fence: optional fence to return
+ *
+ * Open up a stream for HW test
+ */
+int radeon_vce_get_create_msg(struct radeon_device *rdev, int ring,
+                              uint32_t handle, struct radeon_fence **fence)
+{
+        const unsigned ib_size_dw = 1024;
+        struct radeon_ib ib;
+        uint64_t dummy;
+        int i, r;
+
+        r = radeon_ib_get(rdev, ring, &ib, NULL, ib_size_dw * 4);
+        if (r) {
+                DRM_ERROR("radeon: failed to get ib (%d).\n", r);
+                return r;
+        }
+
+        dummy = ib.gpu_addr + 1024;
+
+        /* stitch together an VCE create msg */
+        ib.length_dw = 0;
+        ib.ptr[ib.length_dw++] = 0x0000000c; /* len */
+        ib.ptr[ib.length_dw++] = 0x00000001; /* session cmd */
+        ib.ptr[ib.length_dw++] = handle;
+
+        ib.ptr[ib.length_dw++] = 0x00000030; /* len */
+        ib.ptr[ib.length_dw++] = 0x01000001; /* create cmd */
+        ib.ptr[ib.length_dw++] = 0x00000000;
+        ib.ptr[ib.length_dw++] = 0x00000042;
+        ib.ptr[ib.length_dw++] = 0x0000000a;
+        ib.ptr[ib.length_dw++] = 0x00000001;
+        ib.ptr[ib.length_dw++] = 0x00000080;
+        ib.ptr[ib.length_dw++] = 0x00000060;
+        ib.ptr[ib.length_dw++] = 0x00000100;
+        ib.ptr[ib.length_dw++] = 0x00000100;
+        ib.ptr[ib.length_dw++] = 0x0000000c;
+        ib.ptr[ib.length_dw++] = 0x00000000;
+
+        ib.ptr[ib.length_dw++] = 0x00000014; /* len */
+        ib.ptr[ib.length_dw++] = 0x05000005; /* feedback buffer */
+        ib.ptr[ib.length_dw++] = upper_32_bits(dummy);
+        ib.ptr[ib.length_dw++] = dummy;
+        ib.ptr[ib.length_dw++] = 0x00000001;
+
+        for (i = ib.length_dw; i < ib_size_dw; ++i)
+                ib.ptr[i] = 0x0;
+
+        r = radeon_ib_schedule(rdev, &ib, NULL);
+        if (r) {
+                DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
+        }
+
+        if (fence)
+                *fence = radeon_fence_ref(ib.fence);
+
+        radeon_ib_free(rdev, &ib);
+
+        return r;
+}
+
+/**
+ * radeon_vce_get_destroy_msg - generate a VCE destroy msg
+ *
+ * @rdev: radeon_device pointer
+ * @ring: ring we should submit the msg to
+ * @handle: VCE session handle to use
+ * @fence: optional fence to return
+ *
+ * Close up a stream for HW test or if userspace failed to do so
+ */
+int radeon_vce_get_destroy_msg(struct radeon_device *rdev, int ring,
+                               uint32_t handle, struct radeon_fence **fence)
+{
+        const unsigned ib_size_dw = 1024;
+        struct radeon_ib ib;
+        uint64_t dummy;
+        int i, r;
+
+        r = radeon_ib_get(rdev, ring, &ib, NULL, ib_size_dw * 4);
+        if (r) {
+                DRM_ERROR("radeon: failed to get ib (%d).\n", r);
+                return r;
+        }
+
+        dummy = ib.gpu_addr + 1024;
+
+        /* stitch together an VCE destroy msg */
+        ib.length_dw = 0;
+        ib.ptr[ib.length_dw++] = 0x0000000c; /* len */
+        ib.ptr[ib.length_dw++] = 0x00000001; /* session cmd */
+        ib.ptr[ib.length_dw++] = handle;
+
+        ib.ptr[ib.length_dw++] = 0x00000014; /* len */
+        ib.ptr[ib.length_dw++] = 0x05000005; /* feedback buffer */
+        ib.ptr[ib.length_dw++] = upper_32_bits(dummy);
+        ib.ptr[ib.length_dw++] = dummy;
+        ib.ptr[ib.length_dw++] = 0x00000001;
+
+        ib.ptr[ib.length_dw++] = 0x00000008; /* len */
+        ib.ptr[ib.length_dw++] = 0x02000001; /* destroy cmd */
+
+        for (i = ib.length_dw; i < ib_size_dw; ++i)
+                ib.ptr[i] = 0x0;
+
+        r = radeon_ib_schedule(rdev, &ib, NULL);
+        if (r) {
+                DRM_ERROR("radeon: failed to schedule ib (%d).\n", r);
+        }
+
+        if (fence)
+                *fence = radeon_fence_ref(ib.fence);
+
+        radeon_ib_free(rdev, &ib);
+
+        return r;
+}
+
+/**
+ * radeon_vce_cs_reloc - command submission relocation
+ *
+ * @p: parser context
+ * @lo: address of lower dword
+ * @hi: address of higher dword
+ *
+ * Patch relocation inside command stream with real buffer address
+ */
+int radeon_vce_cs_reloc(struct radeon_cs_parser *p, int lo, int hi)
+{
+        struct radeon_cs_chunk *relocs_chunk;
+        uint64_t offset;
+        unsigned idx;
+
+        relocs_chunk = &p->chunks[p->chunk_relocs_idx];
+        offset = radeon_get_ib_value(p, lo);
+        idx = radeon_get_ib_value(p, hi);
+
+        if (idx >= relocs_chunk->length_dw) {
+                DRM_ERROR("Relocs at %d after relocations chunk end %d !\n",
+                          idx, relocs_chunk->length_dw);
+                return -EINVAL;
+        }
+
+        offset += p->relocs_ptr[(idx / 4)]->gpu_offset;
+
+        p->ib.ptr[lo] = offset & 0xFFFFFFFF;
+        p->ib.ptr[hi] = offset >> 32;
+
+        return 0;
+}
+
+/**
+ * radeon_vce_cs_parse - parse and validate the command stream
+ *
+ * @p: parser context
+ *
+ */
+int radeon_vce_cs_parse(struct radeon_cs_parser *p)
+{
+        uint32_t handle = 0;
+        bool destroy = false;
+        int i, r;
+
+        while (p->idx < p->chunks[p->chunk_ib_idx].length_dw) {
+                uint32_t len = radeon_get_ib_value(p, p->idx);
+                uint32_t cmd = radeon_get_ib_value(p, p->idx + 1);
+
+                if ((len < 8) || (len & 3)) {
+                        DRM_ERROR("invalid VCE command length (%d)!\n", len);
+                        return -EINVAL;
+                }
+
+                switch (cmd) {
+                case 0x00000001: // session
+                        handle = radeon_get_ib_value(p, p->idx + 2);
+                        break;
+
+                case 0x00000002: // task info
+                case 0x01000001: // create
+                case 0x04000001: // config extension
+                case 0x04000002: // pic control
+                case 0x04000005: // rate control
+                case 0x04000007: // motion estimation
+                case 0x04000008: // rdo
+                        break;
+
+                case 0x03000001: // encode
+                        r = radeon_vce_cs_reloc(p, p->idx + 10, p->idx + 9);
+                        if (r)
+                                return r;
+
+                        r = radeon_vce_cs_reloc(p, p->idx + 12, p->idx + 11);
+                        if (r)
+                                return r;
+                        break;
+
+                case 0x02000001: // destroy
+                        destroy = true;
+                        break;
+
+                case 0x05000001: // context buffer
+                case 0x05000004: // video bitstream buffer
+                case 0x05000005: // feedback buffer
+                        r = radeon_vce_cs_reloc(p, p->idx + 3, p->idx + 2);
+                        if (r)
+                                return r;
+                        break;
+
+                default:
+                        DRM_ERROR("invalid VCE command (0x%x)!\n", cmd);
+                        return -EINVAL;
+                }
+
+                p->idx += len / 4;
+        }
+
+        if (destroy) {
+                /* IB contains a destroy msg, free the handle */
+                for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i)
+                        atomic_cmpxchg(&p->rdev->vce.handles[i], handle, 0);
+
+                return 0;
+        }
+
+        /* create or encode, validate the handle */
+        for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) {
+                if (atomic_read(&p->rdev->vce.handles[i]) == handle)
+                        return 0;
+        }
+
+        /* handle not found try to alloc a new one */
+        for (i = 0; i < RADEON_MAX_VCE_HANDLES; ++i) {
+                if (!atomic_cmpxchg(&p->rdev->vce.handles[i], 0, handle)) {
+                        p->rdev->vce.filp[i] = p->filp;
+                        return 0;
+                }
+        }
+
+        DRM_ERROR("No more free VCE handles!\n");
+        return -EINVAL;
+}
+
+/**
+ * radeon_vce_semaphore_emit - emit a semaphore command
+ *
+ * @rdev: radeon_device pointer
+ * @ring: engine to use
+ * @semaphore: address of semaphore
+ * @emit_wait: true=emit wait, false=emit signal
+ *
+ */
+bool radeon_vce_semaphore_emit(struct radeon_device *rdev,
+                               struct radeon_ring *ring,
+                               struct radeon_semaphore *semaphore,
+                               bool emit_wait)
+{
+        uint64_t addr = semaphore->gpu_addr;
+
+        radeon_ring_write(ring, VCE_CMD_SEMAPHORE);
+        radeon_ring_write(ring, (addr >> 3) & 0x000FFFFF);
+        radeon_ring_write(ring, (addr >> 23) & 0x000FFFFF);
+        radeon_ring_write(ring, 0x01003000 | (emit_wait ? 1 : 0));
+        if (!emit_wait)
+                radeon_ring_write(ring, VCE_CMD_END);
+
+        return true;
+}
+
+/**
+ * radeon_vce_ib_execute - execute indirect buffer
+ *
+ * @rdev: radeon_device pointer
+ * @ib: the IB to execute
+ *
+ */
+void radeon_vce_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
+{
+        struct radeon_ring *ring = &rdev->ring[ib->ring];
+        radeon_ring_write(ring, VCE_CMD_IB);
+        radeon_ring_write(ring, ib->gpu_addr);
+        radeon_ring_write(ring, upper_32_bits(ib->gpu_addr));
+        radeon_ring_write(ring, ib->length_dw);
+}
+
+/**
+ * radeon_vce_fence_emit - add a fence command to the ring
+ *
+ * @rdev: radeon_device pointer
+ * @fence: the fence
+ *
+ */
+void radeon_vce_fence_emit(struct radeon_device *rdev,
+                           struct radeon_fence *fence)
+{
+        struct radeon_ring *ring = &rdev->ring[fence->ring];
+        uint32_t addr = rdev->fence_drv[fence->ring].gpu_addr;
+
+        radeon_ring_write(ring, VCE_CMD_FENCE);
+        radeon_ring_write(ring, addr);
+        radeon_ring_write(ring, upper_32_bits(addr));
+        radeon_ring_write(ring, fence->seq);
+        radeon_ring_write(ring, VCE_CMD_TRAP);
+        radeon_ring_write(ring, VCE_CMD_END);
+}
+
+/**
+ * radeon_vce_ring_test - test if VCE ring is working
+ *
+ * @rdev: radeon_device pointer
+ * @ring: the engine to test on
+ *
+ */
+int radeon_vce_ring_test(struct radeon_device *rdev, struct radeon_ring *ring)
+{
+        uint32_t rptr = vce_v1_0_get_rptr(rdev, ring);
+        unsigned i;
+        int r;
+
+        r = radeon_ring_lock(rdev, ring, 16);
+        if (r) {
+                DRM_ERROR("radeon: vce failed to lock ring %d (%d).\n",
+                          ring->idx, r);
+                return r;
+        }
+        radeon_ring_write(ring, VCE_CMD_END);
+        radeon_ring_unlock_commit(rdev, ring);
+
+        for (i = 0; i < rdev->usec_timeout; i++) {
+                if (vce_v1_0_get_rptr(rdev, ring) != rptr)
+                        break;
+                DRM_UDELAY(1);
+        }
+
+        if (i < rdev->usec_timeout) {
+                DRM_INFO("ring test on %d succeeded in %d usecs\n",
+                         ring->idx, i);
+        } else {
+                DRM_ERROR("radeon: ring %d test failed\n",
+                          ring->idx);
+                r = -ETIMEDOUT;
+        }
+
+        return r;
+}
+
+/**
+ * radeon_vce_ib_test - test if VCE IBs are working
+ *
+ * @rdev: radeon_device pointer
+ * @ring: the engine to test on
+ *
+ */
+int radeon_vce_ib_test(struct radeon_device *rdev, struct radeon_ring *ring)
+{
+        struct radeon_fence *fence = NULL;
+        int r;
+
+        r = radeon_vce_get_create_msg(rdev, ring->idx, 1, NULL);
+        if (r) {
+                DRM_ERROR("radeon: failed to get create msg (%d).\n", r);
+                goto error;
+        }
+
+        r = radeon_vce_get_destroy_msg(rdev, ring->idx, 1, &fence);
+        if (r) {
+                DRM_ERROR("radeon: failed to get destroy ib (%d).\n", r);
+                goto error;
+        }
+
+        r = radeon_fence_wait(fence, false);
+        if (r) {
+                DRM_ERROR("radeon: fence wait failed (%d).\n", r);
+        } else {
+                DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+        }
+error:
+        radeon_fence_unref(&fence);
+        return r;
+}
diff --git a/drivers/gpu/drm/radeon/radeon_vm.c b/drivers/gpu/drm/radeon/radeon_vm.c
new file mode 100644
index 000000000000..2aae6ce49d32
--- /dev/null
+++ b/drivers/gpu/drm/radeon/radeon_vm.c
@@ -0,0 +1,966 @@
+/*
+ * Copyright 2008 Advanced Micro Devices, Inc.
+ * Copyright 2008 Red Hat Inc.
+ * Copyright 2009 Jerome Glisse.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Dave Airlie
+ *          Alex Deucher
+ *          Jerome Glisse
+ */
+#include <drm/drmP.h>
+#include <drm/radeon_drm.h>
+#include "radeon.h"
+#include "radeon_trace.h"
+
+/*
+ * GPUVM
+ * GPUVM is similar to the legacy gart on older asics, however
+ * rather than there being a single global gart table
+ * for the entire GPU, there are multiple VM page tables active
+ * at any given time.  The VM page tables can contain a mix
+ * vram pages and system memory pages and system memory pages
+ * can be mapped as snooped (cached system pages) or unsnooped
+ * (uncached system pages).
+ * Each VM has an ID associated with it and there is a page table
+ * associated with each VMID.  When execting a command buffer,
+ * the kernel tells the the ring what VMID to use for that command
+ * buffer.  VMIDs are allocated dynamically as commands are submitted.
+ * The userspace drivers maintain their own address space and the kernel
+ * sets up their pages tables accordingly when they submit their
+ * command buffers and a VMID is assigned.
+ * Cayman/Trinity support up to 8 active VMs at any given time;
+ * SI supports 16.
+ */
+
+/**
+ * radeon_vm_num_pde - return the number of page directory entries
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Calculate the number of page directory entries (cayman+).
+ */
+static unsigned radeon_vm_num_pdes(struct radeon_device *rdev)
+{
+        return rdev->vm_manager.max_pfn >> RADEON_VM_BLOCK_SIZE;
+}
+
+/**
+ * radeon_vm_directory_size - returns the size of the page directory in bytes
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Calculate the size of the page directory in bytes (cayman+).
+ */
+static unsigned radeon_vm_directory_size(struct radeon_device *rdev)
+{
+        return RADEON_GPU_PAGE_ALIGN(radeon_vm_num_pdes(rdev) * 8);
+}
+
+/**
+ * radeon_vm_manager_init - init the vm manager
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Init the vm manager (cayman+).
+ * Returns 0 for success, error for failure.
+ */
+int radeon_vm_manager_init(struct radeon_device *rdev)
+{
+        int r;
+
+        if (!rdev->vm_manager.enabled) {
+                r = radeon_asic_vm_init(rdev);
+                if (r)
+                        return r;
+
+                rdev->vm_manager.enabled = true;
+        }
+        return 0;
+}
+
+/**
+ * radeon_vm_manager_fini - tear down the vm manager
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Tear down the VM manager (cayman+).
+ */
+void radeon_vm_manager_fini(struct radeon_device *rdev)
+{
+        int i;
+
+        if (!rdev->vm_manager.enabled)
+                return;
+
+        for (i = 0; i < RADEON_NUM_VM; ++i)
+                radeon_fence_unref(&rdev->vm_manager.active[i]);
+        radeon_asic_vm_fini(rdev);
+        rdev->vm_manager.enabled = false;
+}
+
+/**
+ * radeon_vm_get_bos - add the vm BOs to a validation list
+ *
+ * @vm: vm providing the BOs
+ * @head: head of validation list
+ *
+ * Add the page directory to the list of BOs to
+ * validate for command submission (cayman+).
+ */
+struct radeon_cs_reloc *radeon_vm_get_bos(struct radeon_device *rdev,
+                                          struct radeon_vm *vm,
+                                          struct list_head *head)
+{
+        struct radeon_cs_reloc *list;
+        unsigned i, idx, size;
+
+        size = (radeon_vm_num_pdes(rdev) + 1) * sizeof(struct radeon_cs_reloc);
+        list = kmalloc(size, GFP_KERNEL);
+        if (!list)
+                return NULL;
+
+        /* add the vm page table to the list */
+        list[0].gobj = NULL;
+        list[0].robj = vm->page_directory;
+        list[0].domain = RADEON_GEM_DOMAIN_VRAM;
+        list[0].alt_domain = RADEON_GEM_DOMAIN_VRAM;
+        list[0].tv.bo = &vm->page_directory->tbo;
+        list[0].tiling_flags = 0;
+        list[0].handle = 0;
+        list_add(&list[0].tv.head, head);
+
+        for (i = 0, idx = 1; i <= vm->max_pde_used; i++) {
+                if (!vm->page_tables[i].bo)
+                        continue;
+
+                list[idx].gobj = NULL;
+                list[idx].robj = vm->page_tables[i].bo;
+                list[idx].domain = RADEON_GEM_DOMAIN_VRAM;
+                list[idx].alt_domain = RADEON_GEM_DOMAIN_VRAM;
+                list[idx].tv.bo = &list[idx].robj->tbo;
+                list[idx].tiling_flags = 0;
+                list[idx].handle = 0;
+                list_add(&list[idx++].tv.head, head);
+        }
+
+        return list;
+}
+
+/**
+ * radeon_vm_grab_id - allocate the next free VMID
+ *
+ * @rdev: radeon_device pointer
+ * @vm: vm to allocate id for
+ * @ring: ring we want to submit job to
+ *
+ * Allocate an id for the vm (cayman+).
+ * Returns the fence we need to sync to (if any).
+ *
+ * Global and local mutex must be locked!
+ */
+struct radeon_fence *radeon_vm_grab_id(struct radeon_device *rdev,
+                                       struct radeon_vm *vm, int ring)
+{
+        struct radeon_fence *best[RADEON_NUM_RINGS] = {};
+        unsigned choices[2] = {};
+        unsigned i;
+
+        /* check if the id is still valid */
+        if (vm->last_id_use && vm->last_id_use == rdev->vm_manager.active[vm->id])
+                return NULL;
+
+        /* we definately need to flush */
+        radeon_fence_unref(&vm->last_flush);
+
+        /* skip over VMID 0, since it is the system VM */
+        for (i = 1; i < rdev->vm_manager.nvm; ++i) {
+                struct radeon_fence *fence = rdev->vm_manager.active[i];
+
+                if (fence == NULL) {
+                        /* found a free one */
+                        vm->id = i;
+                        trace_radeon_vm_grab_id(vm->id, ring);
+                        return NULL;
+                }
+
+                if (radeon_fence_is_earlier(fence, best[fence->ring])) {
+                        best[fence->ring] = fence;
+                        choices[fence->ring == ring ? 0 : 1] = i;
+                }
+        }
+
+        for (i = 0; i < 2; ++i) {
+                if (choices[i]) {
+                        vm->id = choices[i];
+                        trace_radeon_vm_grab_id(vm->id, ring);
+                        return rdev->vm_manager.active[choices[i]];
+                }
+        }
+
+        /* should never happen */
+        BUG();
+        return NULL;
+}
+
+/**
+ * radeon_vm_flush - hardware flush the vm
+ *
+ * @rdev: radeon_device pointer
+ * @vm: vm we want to flush
+ * @ring: ring to use for flush
+ *
+ * Flush the vm (cayman+).
+ *
+ * Global and local mutex must be locked!
+ */
+void radeon_vm_flush(struct radeon_device *rdev,
+                     struct radeon_vm *vm,
+                     int ring)
+{
+        uint64_t pd_addr = radeon_bo_gpu_offset(vm->page_directory);
+
+        /* if we can't remember our last VM flush then flush now! */
+        /* XXX figure out why we have to flush all the time */
+        if (!vm->last_flush || true || pd_addr != vm->pd_gpu_addr) {
+                vm->pd_gpu_addr = pd_addr;
+                radeon_ring_vm_flush(rdev, ring, vm);
+        }
+}
+
+/**
+ * radeon_vm_fence - remember fence for vm
+ *
+ * @rdev: radeon_device pointer
+ * @vm: vm we want to fence
+ * @fence: fence to remember
+ *
+ * Fence the vm (cayman+).
+ * Set the fence used to protect page table and id.
+ *
+ * Global and local mutex must be locked!
+ */
+void radeon_vm_fence(struct radeon_device *rdev,
+                     struct radeon_vm *vm,
+                     struct radeon_fence *fence)
+{
+        radeon_fence_unref(&vm->fence);
+        vm->fence = radeon_fence_ref(fence);
+
+        radeon_fence_unref(&rdev->vm_manager.active[vm->id]);
+        rdev->vm_manager.active[vm->id] = radeon_fence_ref(fence);
+
+        radeon_fence_unref(&vm->last_id_use);
+        vm->last_id_use = radeon_fence_ref(fence);
+
+        /* we just flushed the VM, remember that */
+        if (!vm->last_flush)
+                vm->last_flush = radeon_fence_ref(fence);
+}
+
+/**
+ * radeon_vm_bo_find - find the bo_va for a specific vm & bo
+ *
+ * @vm: requested vm
+ * @bo: requested buffer object
+ *
+ * Find @bo inside the requested vm (cayman+).
+ * Search inside the @bos vm list for the requested vm
+ * Returns the found bo_va or NULL if none is found
+ *
+ * Object has to be reserved!
+ */
+struct radeon_bo_va *radeon_vm_bo_find(struct radeon_vm *vm,
+                                       struct radeon_bo *bo)
+{
+        struct radeon_bo_va *bo_va;
+
+        list_for_each_entry(bo_va, &bo->va, bo_list) {
+                if (bo_va->vm == vm) {
+                        return bo_va;
+                }
+        }
+        return NULL;
+}
+
+/**
+ * radeon_vm_bo_add - add a bo to a specific vm
+ *
+ * @rdev: radeon_device pointer
+ * @vm: requested vm
+ * @bo: radeon buffer object
+ *
+ * Add @bo into the requested vm (cayman+).
+ * Add @bo to the list of bos associated with the vm
+ * Returns newly added bo_va or NULL for failure
+ *
+ * Object has to be reserved!
+ */
+struct radeon_bo_va *radeon_vm_bo_add(struct radeon_device *rdev,
+                                      struct radeon_vm *vm,
+                                      struct radeon_bo *bo)
+{
+        struct radeon_bo_va *bo_va;
+
+        bo_va = kzalloc(sizeof(struct radeon_bo_va), GFP_KERNEL);
+        if (bo_va == NULL) {
+                return NULL;
+        }
+        bo_va->vm = vm;
+        bo_va->bo = bo;
+        bo_va->soffset = 0;
+        bo_va->eoffset = 0;
+        bo_va->flags = 0;
+        bo_va->valid = false;
+        bo_va->ref_count = 1;
+        INIT_LIST_HEAD(&bo_va->bo_list);
+        INIT_LIST_HEAD(&bo_va->vm_list);
+
+        mutex_lock(&vm->mutex);
+        list_add(&bo_va->vm_list, &vm->va);
+        list_add_tail(&bo_va->bo_list, &bo->va);
+        mutex_unlock(&vm->mutex);
+
+        return bo_va;
+}
+
+/**
+ * radeon_vm_clear_bo - initially clear the page dir/table
+ *
+ * @rdev: radeon_device pointer
+ * @bo: bo to clear
+ */
+static int radeon_vm_clear_bo(struct radeon_device *rdev,
+                              struct radeon_bo *bo)
+{
+        struct ttm_validate_buffer tv;
+        struct ww_acquire_ctx ticket;
+        struct list_head head;
+        struct radeon_ib ib;
+        unsigned entries;
+        uint64_t addr;
+        int r;
+
+        memset(&tv, 0, sizeof(tv));
+        tv.bo = &bo->tbo;
+
+        INIT_LIST_HEAD(&head);
+        list_add(&tv.head, &head);
+
+        r = ttm_eu_reserve_buffers(&ticket, &head);
+        if (r)
+                return r;
+
+        r = ttm_bo_validate(&bo->tbo, &bo->placement, true, false);
+        if (r)
+                goto error;
+
+        addr = radeon_bo_gpu_offset(bo);
+        entries = radeon_bo_size(bo) / 8;
+
+        r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib,
+                          NULL, entries * 2 + 64);
+        if (r)
+                goto error;
+
+        ib.length_dw = 0;
+
+        radeon_asic_vm_set_page(rdev, &ib, addr, 0, entries, 0, 0);
+
+        r = radeon_ib_schedule(rdev, &ib, NULL);
+        if (r)
+                goto error;
+
+        ttm_eu_fence_buffer_objects(&ticket, &head, ib.fence);
+        radeon_ib_free(rdev, &ib);
+
+        return 0;
+
+error:
+        ttm_eu_backoff_reservation(&ticket, &head);
+        return r;
+}
+
+/**
+ * radeon_vm_bo_set_addr - set bos virtual address inside a vm
+ *
+ * @rdev: radeon_device pointer
+ * @bo_va: bo_va to store the address
+ * @soffset: requested offset of the buffer in the VM address space
+ * @flags: attributes of pages (read/write/valid/etc.)
+ *
+ * Set offset of @bo_va (cayman+).
+ * Validate and set the offset requested within the vm address space.
+ * Returns 0 for success, error for failure.
+ *
+ * Object has to be reserved!
+ */
+int radeon_vm_bo_set_addr(struct radeon_device *rdev,
+                          struct radeon_bo_va *bo_va,
+                          uint64_t soffset,
+                          uint32_t flags)
+{
+        uint64_t size = radeon_bo_size(bo_va->bo);
+        uint64_t eoffset, last_offset = 0;
+        struct radeon_vm *vm = bo_va->vm;
+        struct radeon_bo_va *tmp;
+        struct list_head *head;
+        unsigned last_pfn, pt_idx;
+        int r;
+
+        if (soffset) {
+                /* make sure object fit at this offset */
+                eoffset = soffset + size;
+                if (soffset >= eoffset) {
+                        return -EINVAL;
+                }
+
+                last_pfn = eoffset / RADEON_GPU_PAGE_SIZE;
+                if (last_pfn > rdev->vm_manager.max_pfn) {
+                        dev_err(rdev->dev, "va above limit (0x%08X > 0x%08X)\n",
+                                last_pfn, rdev->vm_manager.max_pfn);
+                        return -EINVAL;
+                }
+
+        } else {
+                eoffset = last_pfn = 0;
+        }
+
+        mutex_lock(&vm->mutex);
+        head = &vm->va;
+        last_offset = 0;
+        list_for_each_entry(tmp, &vm->va, vm_list) {
+                if (bo_va == tmp) {
+                        /* skip over currently modified bo */
+                        continue;
+                }
+
+                if (soffset >= last_offset && eoffset <= tmp->soffset) {
+                        /* bo can be added before this one */
+                        break;
+                }
+                if (eoffset > tmp->soffset && soffset < tmp->eoffset) {
+                        /* bo and tmp overlap, invalid offset */
+                        dev_err(rdev->dev, "bo %p va 0x%08X conflict with (bo %p 0x%08X 0x%08X)\n",
+                                bo_va->bo, (unsigned)bo_va->soffset, tmp->bo,
+                                (unsigned)tmp->soffset, (unsigned)tmp->eoffset);
+                        mutex_unlock(&vm->mutex);
+                        return -EINVAL;
+                }
+                last_offset = tmp->eoffset;
+                head = &tmp->vm_list;
+        }
+
+        bo_va->soffset = soffset;
+        bo_va->eoffset = eoffset;
+        bo_va->flags = flags;
+        bo_va->valid = false;
+        list_move(&bo_va->vm_list, head);
+
+        soffset = (soffset / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE;
+        eoffset = (eoffset / RADEON_GPU_PAGE_SIZE) >> RADEON_VM_BLOCK_SIZE;
+
+        if (eoffset > vm->max_pde_used)
+                vm->max_pde_used = eoffset;
+
+        radeon_bo_unreserve(bo_va->bo);
+
+        /* walk over the address space and allocate the page tables */
+        for (pt_idx = soffset; pt_idx <= eoffset; ++pt_idx) {
+                struct radeon_bo *pt;
+
+                if (vm->page_tables[pt_idx].bo)
+                        continue;
+
+                /* drop mutex to allocate and clear page table */
+                mutex_unlock(&vm->mutex);
+
+                r = radeon_bo_create(rdev, RADEON_VM_PTE_COUNT * 8,
+                                     RADEON_GPU_PAGE_SIZE, false, 
+                                     RADEON_GEM_DOMAIN_VRAM, NULL, &pt);
+                if (r)
+                        return r;
+
+                r = radeon_vm_clear_bo(rdev, pt);
+                if (r) {
+                        radeon_bo_unref(&pt);
+                        radeon_bo_reserve(bo_va->bo, false);
+                        return r;
+                }
+
+                /* aquire mutex again */
+                mutex_lock(&vm->mutex);
+                if (vm->page_tables[pt_idx].bo) {
+                        /* someone else allocated the pt in the meantime */
+                        mutex_unlock(&vm->mutex);
+                        radeon_bo_unref(&pt);
+                        mutex_lock(&vm->mutex);
+                        continue;
+                }
+
+                vm->page_tables[pt_idx].addr = 0;
+                vm->page_tables[pt_idx].bo = pt;
+        }
+
+        mutex_unlock(&vm->mutex);
+        return radeon_bo_reserve(bo_va->bo, false);
+}
+
+/**
+ * radeon_vm_map_gart - get the physical address of a gart page
+ *
+ * @rdev: radeon_device pointer
+ * @addr: the unmapped addr
+ *
+ * Look up the physical address of the page that the pte resolves
+ * to (cayman+).
+ * Returns the physical address of the page.
+ */
+uint64_t radeon_vm_map_gart(struct radeon_device *rdev, uint64_t addr)
+{
+        uint64_t result;
+
+        /* page table offset */
+        result = rdev->gart.pages_addr[addr >> PAGE_SHIFT];
+
+        /* in case cpu page size != gpu page size*/
+        result |= addr & (~PAGE_MASK);
+
+        return result;
+}
+
+/**
+ * radeon_vm_page_flags - translate page flags to what the hw uses
+ *
+ * @flags: flags comming from userspace
+ *
+ * Translate the flags the userspace ABI uses to hw flags.
+ */
+static uint32_t radeon_vm_page_flags(uint32_t flags)
+{
+        uint32_t hw_flags = 0;
+        hw_flags |= (flags & RADEON_VM_PAGE_VALID) ? R600_PTE_VALID : 0;
+        hw_flags |= (flags & RADEON_VM_PAGE_READABLE) ? R600_PTE_READABLE : 0;
+        hw_flags |= (flags & RADEON_VM_PAGE_WRITEABLE) ? R600_PTE_WRITEABLE : 0;
+        if (flags & RADEON_VM_PAGE_SYSTEM) {
+                hw_flags |= R600_PTE_SYSTEM;
+                hw_flags |= (flags & RADEON_VM_PAGE_SNOOPED) ? R600_PTE_SNOOPED : 0;
+        }
+        return hw_flags;
+}
+
+/**
+ * radeon_vm_update_pdes - make sure that page directory is valid
+ *
+ * @rdev: radeon_device pointer
+ * @vm: requested vm
+ * @start: start of GPU address range
+ * @end: end of GPU address range
+ *
+ * Allocates new page tables if necessary
+ * and updates the page directory (cayman+).
+ * Returns 0 for success, error for failure.
+ *
+ * Global and local mutex must be locked!
+ */
+int radeon_vm_update_page_directory(struct radeon_device *rdev,
+                                    struct radeon_vm *vm)
+{
+        static const uint32_t incr = RADEON_VM_PTE_COUNT * 8;
+
+        uint64_t pd_addr = radeon_bo_gpu_offset(vm->page_directory);
+        uint64_t last_pde = ~0, last_pt = ~0;
+        unsigned count = 0, pt_idx, ndw;
+        struct radeon_ib ib;
+        int r;
+
+        /* padding, etc. */
+        ndw = 64;
+
+        /* assume the worst case */
+        ndw += vm->max_pde_used * 12;
+
+        /* update too big for an IB */
+        if (ndw > 0xfffff)
+                return -ENOMEM;
+
+        r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4);
+        if (r)
+                return r;
+        ib.length_dw = 0;
+
+        /* walk over the address space and update the page directory */
+        for (pt_idx = 0; pt_idx <= vm->max_pde_used; ++pt_idx) {
+                struct radeon_bo *bo = vm->page_tables[pt_idx].bo;
+                uint64_t pde, pt;
+
+                if (bo == NULL)
+                        continue;
+
+                pt = radeon_bo_gpu_offset(bo);
+                if (vm->page_tables[pt_idx].addr == pt)
+                        continue;
+                vm->page_tables[pt_idx].addr = pt;
+
+                pde = pd_addr + pt_idx * 8;
+                if (((last_pde + 8 * count) != pde) ||
+                    ((last_pt + incr * count) != pt)) {
+
+                        if (count) {
+                                radeon_asic_vm_set_page(rdev, &ib, last_pde,
+                                                        last_pt, count, incr,
+                                                        R600_PTE_VALID);
+                        }
+
+                        count = 1;
+                        last_pde = pde;
+                        last_pt = pt;
+                } else {
+                        ++count;
+                }
+        }
+
+        if (count)
+                radeon_asic_vm_set_page(rdev, &ib, last_pde, last_pt, count,
+                                        incr, R600_PTE_VALID);
+
+        if (ib.length_dw != 0) {
+                radeon_semaphore_sync_to(ib.semaphore, vm->last_id_use);
+                r = radeon_ib_schedule(rdev, &ib, NULL);
+                if (r) {
+                        radeon_ib_free(rdev, &ib);
+                        return r;
+                }
+                radeon_fence_unref(&vm->fence);
+                vm->fence = radeon_fence_ref(ib.fence);
+                radeon_fence_unref(&vm->last_flush);
+        }
+        radeon_ib_free(rdev, &ib);
+
+        return 0;
+}
+
+/**
+ * radeon_vm_update_ptes - make sure that page tables are valid
+ *
+ * @rdev: radeon_device pointer
+ * @vm: requested vm
+ * @start: start of GPU address range
+ * @end: end of GPU address range
+ * @dst: destination address to map to
+ * @flags: mapping flags
+ *
+ * Update the page tables in the range @start - @end (cayman+).
+ *
+ * Global and local mutex must be locked!
+ */
+static void radeon_vm_update_ptes(struct radeon_device *rdev,
+                                  struct radeon_vm *vm,
+                                  struct radeon_ib *ib,
+                                  uint64_t start, uint64_t end,
+                                  uint64_t dst, uint32_t flags)
+{
+        static const uint64_t mask = RADEON_VM_PTE_COUNT - 1;
+
+        uint64_t last_pte = ~0, last_dst = ~0;
+        unsigned count = 0;
+        uint64_t addr;
+
+        start = start / RADEON_GPU_PAGE_SIZE;
+        end = end / RADEON_GPU_PAGE_SIZE;
+
+        /* walk over the address space and update the page tables */
+        for (addr = start; addr < end; ) {
+                uint64_t pt_idx = addr >> RADEON_VM_BLOCK_SIZE;
+                unsigned nptes;
+                uint64_t pte;
+
+                if ((addr & ~mask) == (end & ~mask))
+                        nptes = end - addr;
+                else
+                        nptes = RADEON_VM_PTE_COUNT - (addr & mask);
+
+                pte = radeon_bo_gpu_offset(vm->page_tables[pt_idx].bo);
+                pte += (addr & mask) * 8;
+
+                if ((last_pte + 8 * count) != pte) {
+
+                        if (count) {
+                                radeon_asic_vm_set_page(rdev, ib, last_pte,
+                                                        last_dst, count,
+                                                        RADEON_GPU_PAGE_SIZE,
+                                                        flags);
+                        }
+
+                        count = nptes;
+                        last_pte = pte;
+                        last_dst = dst;
+                } else {
+                        count += nptes;
+                }
+
+                addr += nptes;
+                dst += nptes * RADEON_GPU_PAGE_SIZE;
+        }
+
+        if (count) {
+                radeon_asic_vm_set_page(rdev, ib, last_pte,
+                                        last_dst, count,
+                                        RADEON_GPU_PAGE_SIZE, flags);
+        }
+}
+
+/**
+ * radeon_vm_bo_update - map a bo into the vm page table
+ *
+ * @rdev: radeon_device pointer
+ * @vm: requested vm
+ * @bo: radeon buffer object
+ * @mem: ttm mem
+ *
+ * Fill in the page table entries for @bo (cayman+).
+ * Returns 0 for success, -EINVAL for failure.
+ *
+ * Object have to be reserved and mutex must be locked!
+ */
+int radeon_vm_bo_update(struct radeon_device *rdev,
+                        struct radeon_vm *vm,
+                        struct radeon_bo *bo,
+                        struct ttm_mem_reg *mem)
+{
+        struct radeon_ib ib;
+        struct radeon_bo_va *bo_va;
+        unsigned nptes, ndw;
+        uint64_t addr;
+        int r;
+
+        bo_va = radeon_vm_bo_find(vm, bo);
+        if (bo_va == NULL) {
+                dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm);
+                return -EINVAL;
+        }
+
+        if (!bo_va->soffset) {
+                dev_err(rdev->dev, "bo %p don't has a mapping in vm %p\n",
+                        bo, vm);
+                return -EINVAL;
+        }
+
+        if ((bo_va->valid && mem) || (!bo_va->valid && mem == NULL))
+                return 0;
+
+        bo_va->flags &= ~RADEON_VM_PAGE_VALID;
+        bo_va->flags &= ~RADEON_VM_PAGE_SYSTEM;
+        if (mem) {
+                addr = mem->start << PAGE_SHIFT;
+                if (mem->mem_type != TTM_PL_SYSTEM) {
+                        bo_va->flags |= RADEON_VM_PAGE_VALID;
+                        bo_va->valid = true;
+                }
+                if (mem->mem_type == TTM_PL_TT) {
+                        bo_va->flags |= RADEON_VM_PAGE_SYSTEM;
+                } else {
+                        addr += rdev->vm_manager.vram_base_offset;
+                }
+        } else {
+                addr = 0;
+                bo_va->valid = false;
+        }
+
+        trace_radeon_vm_bo_update(bo_va);
+
+        nptes = radeon_bo_ngpu_pages(bo);
+
+        /* padding, etc. */
+        ndw = 64;
+
+        if (RADEON_VM_BLOCK_SIZE > 11)
+                /* reserve space for one header for every 2k dwords */
+                ndw += (nptes >> 11) * 4;
+        else
+                /* reserve space for one header for
+                    every (1 << BLOCK_SIZE) entries */
+                ndw += (nptes >> RADEON_VM_BLOCK_SIZE) * 4;
+
+        /* reserve space for pte addresses */
+        ndw += nptes * 2;
+
+        /* update too big for an IB */
+        if (ndw > 0xfffff)
+                return -ENOMEM;
+
+        r = radeon_ib_get(rdev, R600_RING_TYPE_DMA_INDEX, &ib, NULL, ndw * 4);
+        if (r)
+                return r;
+        ib.length_dw = 0;
+
+        radeon_vm_update_ptes(rdev, vm, &ib, bo_va->soffset, bo_va->eoffset,
+                              addr, radeon_vm_page_flags(bo_va->flags));
+
+        radeon_semaphore_sync_to(ib.semaphore, vm->fence);
+        r = radeon_ib_schedule(rdev, &ib, NULL);
+        if (r) {
+                radeon_ib_free(rdev, &ib);
+                return r;
+        }
+        radeon_fence_unref(&vm->fence);
+        vm->fence = radeon_fence_ref(ib.fence);
+        radeon_ib_free(rdev, &ib);
+        radeon_fence_unref(&vm->last_flush);
+
+        return 0;
+}
+
+/**
+ * radeon_vm_bo_rmv - remove a bo to a specific vm
+ *
+ * @rdev: radeon_device pointer
+ * @bo_va: requested bo_va
+ *
+ * Remove @bo_va->bo from the requested vm (cayman+).
+ * Remove @bo_va->bo from the list of bos associated with the bo_va->vm and
+ * remove the ptes for @bo_va in the page table.
+ * Returns 0 for success.
+ *
+ * Object have to be reserved!
+ */
+int radeon_vm_bo_rmv(struct radeon_device *rdev,
+                     struct radeon_bo_va *bo_va)
+{
+        int r = 0;
+
+        mutex_lock(&bo_va->vm->mutex);
+        if (bo_va->soffset)
+                r = radeon_vm_bo_update(rdev, bo_va->vm, bo_va->bo, NULL);
+
+        list_del(&bo_va->vm_list);
+        mutex_unlock(&bo_va->vm->mutex);
+        list_del(&bo_va->bo_list);
+
+        kfree(bo_va);
+        return r;
+}
+
+/**
+ * radeon_vm_bo_invalidate - mark the bo as invalid
+ *
+ * @rdev: radeon_device pointer
+ * @vm: requested vm
+ * @bo: radeon buffer object
+ *
+ * Mark @bo as invalid (cayman+).
+ */
+void radeon_vm_bo_invalidate(struct radeon_device *rdev,
+                             struct radeon_bo *bo)
+{
+        struct radeon_bo_va *bo_va;
+
+        list_for_each_entry(bo_va, &bo->va, bo_list) {
+                bo_va->valid = false;
+        }
+}
+
+/**
+ * radeon_vm_init - initialize a vm instance
+ *
+ * @rdev: radeon_device pointer
+ * @vm: requested vm
+ *
+ * Init @vm fields (cayman+).
+ */
+int radeon_vm_init(struct radeon_device *rdev, struct radeon_vm *vm)
+{
+        unsigned pd_size, pd_entries, pts_size;
+        int r;
+
+        vm->id = 0;
+        vm->fence = NULL;
+        vm->last_flush = NULL;
+        vm->last_id_use = NULL;
+        mutex_init(&vm->mutex);
+        INIT_LIST_HEAD(&vm->va);
+
+        pd_size = radeon_vm_directory_size(rdev);
+        pd_entries = radeon_vm_num_pdes(rdev);
+
+        /* allocate page table array */
+        pts_size = pd_entries * sizeof(struct radeon_vm_pt);
+        vm->page_tables = kzalloc(pts_size, GFP_KERNEL);
+        if (vm->page_tables == NULL) {
+                DRM_ERROR("Cannot allocate memory for page table array\n");
+                return -ENOMEM;
+        }
+
+        r = radeon_bo_create(rdev, pd_size, RADEON_VM_PTB_ALIGN_SIZE, false,
+                             RADEON_GEM_DOMAIN_VRAM, NULL,
+                             &vm->page_directory);
+        if (r)
+                return r;
+
+        r = radeon_vm_clear_bo(rdev, vm->page_directory);
+        if (r) {
+                radeon_bo_unref(&vm->page_directory);
+                vm->page_directory = NULL;
+                return r;
+        }
+
+        return 0;
+}
+
+/**
+ * radeon_vm_fini - tear down a vm instance
+ *
+ * @rdev: radeon_device pointer
+ * @vm: requested vm
+ *
+ * Tear down @vm (cayman+).
+ * Unbind the VM and remove all bos from the vm bo list
+ */
+void radeon_vm_fini(struct radeon_device *rdev, struct radeon_vm *vm)
+{
+        struct radeon_bo_va *bo_va, *tmp;
+        int i, r;
+
+        if (!list_empty(&vm->va)) {
+                dev_err(rdev->dev, "still active bo inside vm\n");
+        }
+        list_for_each_entry_safe(bo_va, tmp, &vm->va, vm_list) {
+                list_del_init(&bo_va->vm_list);
+                r = radeon_bo_reserve(bo_va->bo, false);
+                if (!r) {
+                        list_del_init(&bo_va->bo_list);
+                        radeon_bo_unreserve(bo_va->bo);
+                        kfree(bo_va);
+                }
+        }
+
+
+        for (i = 0; i < radeon_vm_num_pdes(rdev); i++)
+                radeon_bo_unref(&vm->page_tables[i].bo);
+        kfree(vm->page_tables);
+
+        radeon_bo_unref(&vm->page_directory);
+
+        radeon_fence_unref(&vm->fence);
+        radeon_fence_unref(&vm->last_flush);
+        radeon_fence_unref(&vm->last_id_use);
+
+        mutex_destroy(&vm->mutex);
+}
diff --git a/drivers/gpu/drm/radeon/rs780_dpm.c b/drivers/gpu/drm/radeon/rs780_dpm.c
index 8512085b0aef..02f7710de470 100644
--- a/drivers/gpu/drm/radeon/rs780_dpm.c
+++ b/drivers/gpu/drm/radeon/rs780_dpm.c
@@ -807,9 +807,6 @@ static int rs780_parse_power_table(struct radeon_device *rdev)
                                   power_info->pplib.ucNumStates, GFP_KERNEL);
         if (!rdev->pm.dpm.ps)
                 return -ENOMEM;
-        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
-        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
-        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
 
         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
                 power_state = (union pplib_power_state *)
@@ -859,6 +856,10 @@ int rs780_dpm_init(struct radeon_device *rdev)
                 return -ENOMEM;
         rdev->pm.dpm.priv = pi;
 
+        ret = r600_get_platform_caps(rdev);
+        if (ret)
+                return ret;
+
         ret = rs780_parse_power_table(rdev);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/radeon/rv6xx_dpm.c b/drivers/gpu/drm/radeon/rv6xx_dpm.c
index bebf31c4d841..e7045b085715 100644
--- a/drivers/gpu/drm/radeon/rv6xx_dpm.c
+++ b/drivers/gpu/drm/radeon/rv6xx_dpm.c
@@ -1891,9 +1891,6 @@ static int rv6xx_parse_power_table(struct radeon_device *rdev)
                                   power_info->pplib.ucNumStates, GFP_KERNEL);
         if (!rdev->pm.dpm.ps)
                 return -ENOMEM;
-        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
-        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
-        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
 
         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
                 power_state = (union pplib_power_state *)
@@ -1943,6 +1940,10 @@ int rv6xx_dpm_init(struct radeon_device *rdev)
                 return -ENOMEM;
         rdev->pm.dpm.priv = pi;
 
+        ret = r600_get_platform_caps(rdev);
+        if (ret)
+                return ret;
+
         ret = rv6xx_parse_power_table(rdev);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/radeon/rv770_dpm.c b/drivers/gpu/drm/radeon/rv770_dpm.c
index b5f63f5e22a3..da041a43d82e 100644
--- a/drivers/gpu/drm/radeon/rv770_dpm.c
+++ b/drivers/gpu/drm/radeon/rv770_dpm.c
@@ -2281,9 +2281,6 @@ int rv7xx_parse_power_table(struct radeon_device *rdev)
                                   power_info->pplib.ucNumStates, GFP_KERNEL);
         if (!rdev->pm.dpm.ps)
                 return -ENOMEM;
-        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
-        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
-        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
 
         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
                 power_state = (union pplib_power_state *)
@@ -2361,6 +2358,10 @@ int rv770_dpm_init(struct radeon_device *rdev)
         pi->min_vddc_in_table = 0;
         pi->max_vddc_in_table = 0;
 
+        ret = r600_get_platform_caps(rdev);
+        if (ret)
+                return ret;
+
         ret = rv7xx_parse_power_table(rdev);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c
index 9a124d0608b3..d589475fe9e6 100644
--- a/drivers/gpu/drm/radeon/si.c
+++ b/drivers/gpu/drm/radeon/si.c
@@ -3434,8 +3434,6 @@ static int si_cp_resume(struct radeon_device *rdev)
 
         WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
 
-        ring->rptr = RREG32(CP_RB0_RPTR);
-
         /* ring1  - compute only */
         /* Set ring buffer size */
         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
@@ -3460,8 +3458,6 @@ static int si_cp_resume(struct radeon_device *rdev)
 
         WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
 
-        ring->rptr = RREG32(CP_RB1_RPTR);
-
         /* ring2 - compute only */
         /* Set ring buffer size */
         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
@@ -3486,8 +3482,6 @@ static int si_cp_resume(struct radeon_device *rdev)
 
         WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
 
-        ring->rptr = RREG32(CP_RB2_RPTR);
-
         /* start the rings */
         si_cp_start(rdev);
         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
@@ -3872,11 +3866,9 @@ bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
         if (!(reset_mask & (RADEON_RESET_GFX |
                             RADEON_RESET_COMPUTE |
                             RADEON_RESET_CP))) {
-                radeon_ring_lockup_update(ring);
+                radeon_ring_lockup_update(rdev, ring);
                 return false;
         }
-        /* force CP activities */
-        radeon_ring_force_activity(rdev, ring);
         return radeon_ring_test_lockup(rdev, ring);
 }
 
diff --git a/drivers/gpu/drm/radeon/si_dma.c b/drivers/gpu/drm/radeon/si_dma.c
index 59be2cfcbb47..cf0fdad8c278 100644
--- a/drivers/gpu/drm/radeon/si_dma.c
+++ b/drivers/gpu/drm/radeon/si_dma.c
@@ -49,11 +49,9 @@ bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
                 mask = RADEON_RESET_DMA1;
 
         if (!(reset_mask & mask)) {
-                radeon_ring_lockup_update(ring);
+                radeon_ring_lockup_update(rdev, ring);
                 return false;
         }
-        /* force ring activities */
-        radeon_ring_force_activity(rdev, ring);
         return radeon_ring_test_lockup(rdev, ring);
 }
 
diff --git a/drivers/gpu/drm/radeon/si_dpm.c b/drivers/gpu/drm/radeon/si_dpm.c
index 0a2f5b4bca43..9a3567bedaae 100644
--- a/drivers/gpu/drm/radeon/si_dpm.c
+++ b/drivers/gpu/drm/radeon/si_dpm.c
@@ -6271,9 +6271,6 @@ static int si_parse_power_table(struct radeon_device *rdev)
         if (!rdev->pm.dpm.ps)
                 return -ENOMEM;
         power_state_offset = (u8 *)state_array->states;
-        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
-        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
-        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
         for (i = 0; i < state_array->ucNumEntries; i++) {
                 u8 *idx;
                 power_state = (union pplib_power_state *)power_state_offset;
@@ -6350,6 +6347,10 @@ int si_dpm_init(struct radeon_device *rdev)
         pi->min_vddc_in_table = 0;
         pi->max_vddc_in_table = 0;
 
+        ret = r600_get_platform_caps(rdev);
+        if (ret)
+                return ret;
+
         ret = si_parse_power_table(rdev);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/radeon/sid.h b/drivers/gpu/drm/radeon/sid.h
index 9239a6d29128..683532f84931 100644
--- a/drivers/gpu/drm/radeon/sid.h
+++ b/drivers/gpu/drm/radeon/sid.h
@@ -1798,4 +1798,51 @@
 #define DMA_PACKET_CONSTANT_FILL                          0xd
 #define DMA_PACKET_NOP                                    0xf
 
+#define VCE_STATUS                                      0x20004
+#define VCE_VCPU_CNTL                                   0x20014
+#define         VCE_CLK_EN                              (1 << 0)
+#define VCE_VCPU_CACHE_OFFSET0                          0x20024
+#define VCE_VCPU_CACHE_SIZE0                            0x20028
+#define VCE_VCPU_CACHE_OFFSET1                          0x2002c
+#define VCE_VCPU_CACHE_SIZE1                            0x20030
+#define VCE_VCPU_CACHE_OFFSET2                          0x20034
+#define VCE_VCPU_CACHE_SIZE2                            0x20038
+#define VCE_SOFT_RESET                                  0x20120
+#define         VCE_ECPU_SOFT_RESET                     (1 << 0)
+#define         VCE_FME_SOFT_RESET                      (1 << 2)
+#define VCE_RB_BASE_LO2                                 0x2016c
+#define VCE_RB_BASE_HI2                                 0x20170
+#define VCE_RB_SIZE2                                    0x20174
+#define VCE_RB_RPTR2                                    0x20178
+#define VCE_RB_WPTR2                                    0x2017c
+#define VCE_RB_BASE_LO                                  0x20180
+#define VCE_RB_BASE_HI                                  0x20184
+#define VCE_RB_SIZE                                     0x20188
+#define VCE_RB_RPTR                                     0x2018c
+#define VCE_RB_WPTR                                     0x20190
+#define VCE_CLOCK_GATING_A                              0x202f8
+#define VCE_CLOCK_GATING_B                              0x202fc
+#define VCE_UENC_CLOCK_GATING                           0x205bc
+#define VCE_UENC_REG_CLOCK_GATING                       0x205c0
+#define VCE_FW_REG_STATUS                               0x20e10
+#       define VCE_FW_REG_STATUS_BUSY                   (1 << 0)
+#       define VCE_FW_REG_STATUS_PASS                   (1 << 3)
+#       define VCE_FW_REG_STATUS_DONE                   (1 << 11)
+#define VCE_LMI_FW_START_KEYSEL                         0x20e18
+#define VCE_LMI_FW_PERIODIC_CTRL                        0x20e20
+#define VCE_LMI_CTRL2                                   0x20e74
+#define VCE_LMI_CTRL                                    0x20e98
+#define VCE_LMI_VM_CTRL                                 0x20ea0
+#define VCE_LMI_SWAP_CNTL                               0x20eb4
+#define VCE_LMI_SWAP_CNTL1                              0x20eb8
+#define VCE_LMI_CACHE_CTRL                              0x20ef4
+
+#define VCE_CMD_NO_OP                                   0x00000000
+#define VCE_CMD_END                                     0x00000001
+#define VCE_CMD_IB                                      0x00000002
+#define VCE_CMD_FENCE                                   0x00000003
+#define VCE_CMD_TRAP                                    0x00000004
+#define VCE_CMD_IB_AUTO                                 0x00000005
+#define VCE_CMD_SEMAPHORE                               0x00000006
+
 #endif
diff --git a/drivers/gpu/drm/radeon/sumo_dpm.c b/drivers/gpu/drm/radeon/sumo_dpm.c
index 8b47b3cd0357..3f0e8d7b8dbe 100644
--- a/drivers/gpu/drm/radeon/sumo_dpm.c
+++ b/drivers/gpu/drm/radeon/sumo_dpm.c
@@ -1484,9 +1484,6 @@ static int sumo_parse_power_table(struct radeon_device *rdev)
         if (!rdev->pm.dpm.ps)
                 return -ENOMEM;
         power_state_offset = (u8 *)state_array->states;
-        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
-        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
-        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
         for (i = 0; i < state_array->ucNumEntries; i++) {
                 u8 *idx;
                 power_state = (union pplib_power_state *)power_state_offset;
@@ -1772,6 +1769,10 @@ int sumo_dpm_init(struct radeon_device *rdev)
 
         sumo_construct_boot_and_acpi_state(rdev);
 
+        ret = r600_get_platform_caps(rdev);
+        if (ret)
+                return ret;
+
         ret = sumo_parse_power_table(rdev);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/radeon/trinity_dpm.c b/drivers/gpu/drm/radeon/trinity_dpm.c
index 2da0e17eb960..2a2822c03329 100644
--- a/drivers/gpu/drm/radeon/trinity_dpm.c
+++ b/drivers/gpu/drm/radeon/trinity_dpm.c
@@ -1694,9 +1694,6 @@ static int trinity_parse_power_table(struct radeon_device *rdev)
         if (!rdev->pm.dpm.ps)
                 return -ENOMEM;
         power_state_offset = (u8 *)state_array->states;
-        rdev->pm.dpm.platform_caps = le32_to_cpu(power_info->pplib.ulPlatformCaps);
-        rdev->pm.dpm.backbias_response_time = le16_to_cpu(power_info->pplib.usBackbiasTime);
-        rdev->pm.dpm.voltage_response_time = le16_to_cpu(power_info->pplib.usVoltageTime);
         for (i = 0; i < state_array->ucNumEntries; i++) {
                 u8 *idx;
                 power_state = (union pplib_power_state *)power_state_offset;
@@ -1895,6 +1892,10 @@ int trinity_dpm_init(struct radeon_device *rdev)
 
         trinity_construct_boot_state(rdev);
 
+        ret = r600_get_platform_caps(rdev);
+        if (ret)
+                return ret;
+
         ret = trinity_parse_power_table(rdev);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/radeon/uvd_v1_0.c b/drivers/gpu/drm/radeon/uvd_v1_0.c
index d4a68af1a279..0a243f0e5d68 100644
--- a/drivers/gpu/drm/radeon/uvd_v1_0.c
+++ b/drivers/gpu/drm/radeon/uvd_v1_0.c
@@ -262,7 +262,7 @@ int uvd_v1_0_start(struct radeon_device *rdev)
         /* Initialize the ring buffer's read and write pointers */
         WREG32(UVD_RBC_RB_RPTR, 0x0);
 
-        ring->wptr = ring->rptr = RREG32(UVD_RBC_RB_RPTR);
+        ring->wptr = RREG32(UVD_RBC_RB_RPTR);
         WREG32(UVD_RBC_RB_WPTR, ring->wptr);
 
         /* set the ring address */
diff --git a/drivers/gpu/drm/radeon/vce_v1_0.c b/drivers/gpu/drm/radeon/vce_v1_0.c
new file mode 100644
index 000000000000..b44d9c842f7b
--- /dev/null
+++ b/drivers/gpu/drm/radeon/vce_v1_0.c
@@ -0,0 +1,187 @@
+/*
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * Authors: Christian König <christian.koenig@amd.com>
+ */
+
+#include <linux/firmware.h>
+#include <drm/drmP.h>
+#include "radeon.h"
+#include "radeon_asic.h"
+#include "sid.h"
+
+/**
+ * vce_v1_0_get_rptr - get read pointer
+ *
+ * @rdev: radeon_device pointer
+ * @ring: radeon_ring pointer
+ *
+ * Returns the current hardware read pointer
+ */
+uint32_t vce_v1_0_get_rptr(struct radeon_device *rdev,
+                           struct radeon_ring *ring)
+{
+        if (ring->idx == TN_RING_TYPE_VCE1_INDEX)
+                return RREG32(VCE_RB_RPTR);
+        else
+                return RREG32(VCE_RB_RPTR2);
+}
+
+/**
+ * vce_v1_0_get_wptr - get write pointer
+ *
+ * @rdev: radeon_device pointer
+ * @ring: radeon_ring pointer
+ *
+ * Returns the current hardware write pointer
+ */
+uint32_t vce_v1_0_get_wptr(struct radeon_device *rdev,
+                           struct radeon_ring *ring)
+{
+        if (ring->idx == TN_RING_TYPE_VCE1_INDEX)
+                return RREG32(VCE_RB_WPTR);
+        else
+                return RREG32(VCE_RB_WPTR2);
+}
+
+/**
+ * vce_v1_0_set_wptr - set write pointer
+ *
+ * @rdev: radeon_device pointer
+ * @ring: radeon_ring pointer
+ *
+ * Commits the write pointer to the hardware
+ */
+void vce_v1_0_set_wptr(struct radeon_device *rdev,
+                       struct radeon_ring *ring)
+{
+        if (ring->idx == TN_RING_TYPE_VCE1_INDEX)
+                WREG32(VCE_RB_WPTR, ring->wptr);
+        else
+                WREG32(VCE_RB_WPTR2, ring->wptr);
+}
+
+/**
+ * vce_v1_0_start - start VCE block
+ *
+ * @rdev: radeon_device pointer
+ *
+ * Setup and start the VCE block
+ */
+int vce_v1_0_start(struct radeon_device *rdev)
+{
+        struct radeon_ring *ring;
+        int i, j, r;
+
+        /* set BUSY flag */
+        WREG32_P(VCE_STATUS, 1, ~1);
+
+        ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
+        WREG32(VCE_RB_RPTR, ring->wptr);
+        WREG32(VCE_RB_WPTR, ring->wptr);
+        WREG32(VCE_RB_BASE_LO, ring->gpu_addr);
+        WREG32(VCE_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
+        WREG32(VCE_RB_SIZE, ring->ring_size / 4);
+
+        ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
+        WREG32(VCE_RB_RPTR2, ring->wptr);
+        WREG32(VCE_RB_WPTR2, ring->wptr);
+        WREG32(VCE_RB_BASE_LO2, ring->gpu_addr);
+        WREG32(VCE_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
+        WREG32(VCE_RB_SIZE2, ring->ring_size / 4);
+
+        WREG32_P(VCE_VCPU_CNTL, VCE_CLK_EN, ~VCE_CLK_EN);
+
+        WREG32_P(VCE_SOFT_RESET,
+                 VCE_ECPU_SOFT_RESET |
+                 VCE_FME_SOFT_RESET, ~(
+                 VCE_ECPU_SOFT_RESET |
+                 VCE_FME_SOFT_RESET));
+
+        mdelay(100);
+
+        WREG32_P(VCE_SOFT_RESET, 0, ~(
+                 VCE_ECPU_SOFT_RESET |
+                 VCE_FME_SOFT_RESET));
+
+        for (i = 0; i < 10; ++i) {
+                uint32_t status;
+                for (j = 0; j < 100; ++j) {
+                        status = RREG32(VCE_STATUS);
+                        if (status & 2)
+                                break;
+                        mdelay(10);
+                }
+                r = 0;
+                if (status & 2)
+                        break;
+
+                DRM_ERROR("VCE not responding, trying to reset the ECPU!!!\n");
+                WREG32_P(VCE_SOFT_RESET, VCE_ECPU_SOFT_RESET, ~VCE_ECPU_SOFT_RESET);
+                mdelay(10);
+                WREG32_P(VCE_SOFT_RESET, 0, ~VCE_ECPU_SOFT_RESET);
+                mdelay(10);
+                r = -1;
+        }
+
+        /* clear BUSY flag */
+        WREG32_P(VCE_STATUS, 0, ~1);
+
+        if (r) {
+                DRM_ERROR("VCE not responding, giving up!!!\n");
+                return r;
+        }
+
+        return 0;
+}
+
+int vce_v1_0_init(struct radeon_device *rdev)
+{
+        struct radeon_ring *ring;
+        int r;
+
+        r = vce_v1_0_start(rdev);
+        if (r)
+                return r;
+
+        ring = &rdev->ring[TN_RING_TYPE_VCE1_INDEX];
+        ring->ready = true;
+        r = radeon_ring_test(rdev, TN_RING_TYPE_VCE1_INDEX, ring);
+        if (r) {
+                ring->ready = false;
+                return r;
+        }
+
+        ring = &rdev->ring[TN_RING_TYPE_VCE2_INDEX];
+        ring->ready = true;
+        r = radeon_ring_test(rdev, TN_RING_TYPE_VCE2_INDEX, ring);
+        if (r) {
+                ring->ready = false;
+                return r;
+        }
+
+        DRM_INFO("VCE initialized successfully.\n");
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/radeon/vce_v2_0.c b/drivers/gpu/drm/radeon/vce_v2_0.c
new file mode 100644
index 000000000000..1ac7bb825a1b
--- /dev/null
+++ b/drivers/gpu/drm/radeon/vce_v2_0.c
@@ -0,0 +1,181 @@
+/*
+ * Copyright 2013 Advanced Micro Devices, Inc.
+ * All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sub license, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
+ * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
+ * USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial portions
+ * of the Software.
+ *
+ * Authors: Christian König <christian.koenig@amd.com>
+ */
+
+#include <linux/firmware.h>
+#include <drm/drmP.h>
+#include "radeon.h"
+#include "radeon_asic.h"
+#include "cikd.h"
+
+static void vce_v2_0_set_sw_cg(struct radeon_device *rdev, bool gated)
+{
+        u32 tmp;
+
+        if (gated) {
+                tmp = RREG32(VCE_CLOCK_GATING_B);
+                tmp |= 0xe70000;
+                WREG32(VCE_CLOCK_GATING_B, tmp);
+
+                tmp = RREG32(VCE_UENC_CLOCK_GATING);
+                tmp |= 0xff000000;
+                WREG32(VCE_UENC_CLOCK_GATING, tmp);
+
+                tmp = RREG32(VCE_UENC_REG_CLOCK_GATING);
+                tmp &= ~0x3fc;
+                WREG32(VCE_UENC_REG_CLOCK_GATING, tmp);
+
+                WREG32(VCE_CGTT_CLK_OVERRIDE, 0);
+    } else {
+                tmp = RREG32(VCE_CLOCK_GATING_B);
+                tmp |= 0xe7;
+                tmp &= ~0xe70000;
+                WREG32(VCE_CLOCK_GATING_B, tmp);
+
+                tmp = RREG32(VCE_UENC_CLOCK_GATING);
+                tmp |= 0x1fe000;
+                tmp &= ~0xff000000;
+                WREG32(VCE_UENC_CLOCK_GATING, tmp);
+
+                tmp = RREG32(VCE_UENC_REG_CLOCK_GATING);
+                tmp |= 0x3fc;
+                WREG32(VCE_UENC_REG_CLOCK_GATING, tmp);
+        }
+}
+
+static void vce_v2_0_set_dyn_cg(struct radeon_device *rdev, bool gated)
+{
+        u32 orig, tmp;
+
+        tmp = RREG32(VCE_CLOCK_GATING_B);
+        tmp &= ~0x00060006;
+        if (gated) {
+                tmp |= 0xe10000;
+        } else {
+                tmp |= 0xe1;
+                tmp &= ~0xe10000;
+        }
+        WREG32(VCE_CLOCK_GATING_B, tmp);
+
+        orig = tmp = RREG32(VCE_UENC_CLOCK_GATING);
+        tmp &= ~0x1fe000;
+        tmp &= ~0xff000000;
+        if (tmp != orig)
+                WREG32(VCE_UENC_CLOCK_GATING, tmp);
+
+        orig = tmp = RREG32(VCE_UENC_REG_CLOCK_GATING);
+        tmp &= ~0x3fc;
+        if (tmp != orig)
+                WREG32(VCE_UENC_REG_CLOCK_GATING, tmp);
+
+        if (gated)
+                WREG32(VCE_CGTT_CLK_OVERRIDE, 0);
+}
+
+static void vce_v2_0_disable_cg(struct radeon_device *rdev)
+{
+        WREG32(VCE_CGTT_CLK_OVERRIDE, 7);
+}
+
+void vce_v2_0_enable_mgcg(struct radeon_device *rdev, bool enable)
+{
+        bool sw_cg = false;
+
+        if (enable && (rdev->cg_flags & RADEON_CG_SUPPORT_VCE_MGCG)) {
+                if (sw_cg)
+                        vce_v2_0_set_sw_cg(rdev, true);
+                else
+                        vce_v2_0_set_dyn_cg(rdev, true);
+        } else {
+                vce_v2_0_disable_cg(rdev);
+
+                if (sw_cg)
+                        vce_v2_0_set_sw_cg(rdev, false);
+                else
+                        vce_v2_0_set_dyn_cg(rdev, false);
+        }
+}
+
+static void vce_v2_0_init_cg(struct radeon_device *rdev)
+{
+        u32 tmp;
+
+        tmp = RREG32(VCE_CLOCK_GATING_A);
+        tmp &= ~(CGC_CLK_GATE_DLY_TIMER_MASK | CGC_CLK_GATER_OFF_DLY_TIMER_MASK);
+        tmp |= (CGC_CLK_GATE_DLY_TIMER(0) | CGC_CLK_GATER_OFF_DLY_TIMER(4));
+        tmp |= CGC_UENC_WAIT_AWAKE;
+        WREG32(VCE_CLOCK_GATING_A, tmp);
+
+        tmp = RREG32(VCE_UENC_CLOCK_GATING);
+        tmp &= ~(CLOCK_ON_DELAY_MASK | CLOCK_OFF_DELAY_MASK);
+        tmp |= (CLOCK_ON_DELAY(0) | CLOCK_OFF_DELAY(4));
+        WREG32(VCE_UENC_CLOCK_GATING, tmp);
+
+        tmp = RREG32(VCE_CLOCK_GATING_B);
+        tmp |= 0x10;
+        tmp &= ~0x100000;
+        WREG32(VCE_CLOCK_GATING_B, tmp);
+}
+
+int vce_v2_0_resume(struct radeon_device *rdev)
+{
+        uint64_t addr = rdev->vce.gpu_addr;
+        uint32_t size;
+
+        WREG32_P(VCE_CLOCK_GATING_A, 0, ~(1 << 16));
+        WREG32_P(VCE_UENC_CLOCK_GATING, 0x1FF000, ~0xFF9FF000);
+        WREG32_P(VCE_UENC_REG_CLOCK_GATING, 0x3F, ~0x3F);
+        WREG32(VCE_CLOCK_GATING_B, 0xf7);
+
+        WREG32(VCE_LMI_CTRL, 0x00398000);
+        WREG32_P(VCE_LMI_CACHE_CTRL, 0x0, ~0x1);
+        WREG32(VCE_LMI_SWAP_CNTL, 0);
+        WREG32(VCE_LMI_SWAP_CNTL1, 0);
+        WREG32(VCE_LMI_VM_CTRL, 0);
+
+        size = RADEON_GPU_PAGE_ALIGN(rdev->vce_fw->size);
+        WREG32(VCE_VCPU_CACHE_OFFSET0, addr & 0x7fffffff);
+        WREG32(VCE_VCPU_CACHE_SIZE0, size);
+
+        addr += size;
+        size = RADEON_VCE_STACK_SIZE;
+        WREG32(VCE_VCPU_CACHE_OFFSET1, addr & 0x7fffffff);
+        WREG32(VCE_VCPU_CACHE_SIZE1, size);
+
+        addr += size;
+        size = RADEON_VCE_HEAP_SIZE;
+        WREG32(VCE_VCPU_CACHE_OFFSET2, addr & 0x7fffffff);
+        WREG32(VCE_VCPU_CACHE_SIZE2, size);
+
+        WREG32_P(VCE_LMI_CTRL2, 0x0, ~0x100);
+
+        WREG32_P(VCE_SYS_INT_EN, VCE_SYS_INT_TRAP_INTERRUPT_EN,
+                 ~VCE_SYS_INT_TRAP_INTERRUPT_EN);
+
+        vce_v2_0_init_cg(rdev);
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
index fbf4be316d0b..299267db2898 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
@@ -299,7 +299,7 @@ static void rcar_du_crtc_update_base(struct rcar_du_crtc *rcrtc)
 {
         struct drm_crtc *crtc = &rcrtc->crtc;
 
-        rcar_du_plane_compute_base(rcrtc->plane, crtc->fb);
+        rcar_du_plane_compute_base(rcrtc->plane, crtc->primary->fb);
         rcar_du_plane_update_base(rcrtc->plane);
 }
 
@@ -358,10 +358,10 @@ static int rcar_du_crtc_mode_set(struct drm_crtc *crtc,
         const struct rcar_du_format_info *format;
         int ret;
 
-        format = rcar_du_format_info(crtc->fb->pixel_format);
+        format = rcar_du_format_info(crtc->primary->fb->pixel_format);
         if (format == NULL) {
                 dev_dbg(rcdu->dev, "mode_set: unsupported format %08x\n",
-                        crtc->fb->pixel_format);
+                        crtc->primary->fb->pixel_format);
                 ret = -EINVAL;
                 goto error;
         }
@@ -377,7 +377,7 @@ static int rcar_du_crtc_mode_set(struct drm_crtc *crtc,
         rcrtc->plane->width = mode->hdisplay;
         rcrtc->plane->height = mode->vdisplay;
 
-        rcar_du_plane_compute_base(rcrtc->plane, crtc->fb);
+        rcar_du_plane_compute_base(rcrtc->plane, crtc->primary->fb);
 
         rcrtc->outputs = 0;
 
@@ -510,7 +510,7 @@ static int rcar_du_crtc_page_flip(struct drm_crtc *crtc,
         }
         spin_unlock_irqrestore(&dev->event_lock, flags);
 
-        crtc->fb = fb;
+        crtc->primary->fb = fb;
         rcar_du_crtc_update_base(rcrtc);
 
         if (event) {
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_kms.c b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
index fbeabd9a281f..a87edfac111f 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_kms.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
@@ -248,7 +248,10 @@ int rcar_du_modeset_init(struct rcar_du_device *rcdu)
                         continue;
                 }
 
-                rcar_du_encoder_init(rcdu, pdata->type, pdata->output, pdata);
+                ret = rcar_du_encoder_init(rcdu, pdata->type, pdata->output,
+                                           pdata);
+                if (ret < 0)
+                        return ret;
         }
 
         /* Set the possible CRTCs and possible clones. There's always at least
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
index 0428076f1ce8..e9e5e6d368cc 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
@@ -173,7 +173,7 @@ static void shmob_drm_crtc_start(struct shmob_drm_crtc *scrtc)
         if (scrtc->started)
                 return;
 
-        format = shmob_drm_format_info(crtc->fb->pixel_format);
+        format = shmob_drm_format_info(crtc->primary->fb->pixel_format);
         if (WARN_ON(format == NULL))
                 return;
 
@@ -247,7 +247,7 @@ static void shmob_drm_crtc_start(struct shmob_drm_crtc *scrtc)
         lcdc_write(sdev, LDDDSR, value);
 
         /* Setup planes. */
-        list_for_each_entry(plane, &dev->mode_config.plane_list, head) {
+        drm_for_each_legacy_plane(plane, &dev->mode_config.plane_list) {
                 if (plane->crtc == crtc)
                         shmob_drm_plane_setup(plane);
         }
@@ -303,7 +303,7 @@ static void shmob_drm_crtc_compute_base(struct shmob_drm_crtc *scrtc,
                                         int x, int y)
 {
         struct drm_crtc *crtc = &scrtc->crtc;
-        struct drm_framebuffer *fb = crtc->fb;
+        struct drm_framebuffer *fb = crtc->primary->fb;
         struct shmob_drm_device *sdev = crtc->dev->dev_private;
         struct drm_gem_cma_object *gem;
         unsigned int bpp;
@@ -382,15 +382,15 @@ static int shmob_drm_crtc_mode_set(struct drm_crtc *crtc,
         const struct shmob_drm_format_info *format;
         void *cache;
 
-        format = shmob_drm_format_info(crtc->fb->pixel_format);
+        format = shmob_drm_format_info(crtc->primary->fb->pixel_format);
         if (format == NULL) {
                 dev_dbg(sdev->dev, "mode_set: unsupported format %08x\n",
-                        crtc->fb->pixel_format);
+                        crtc->primary->fb->pixel_format);
                 return -EINVAL;
         }
 
         scrtc->format = format;
-        scrtc->line_size = crtc->fb->pitches[0];
+        scrtc->line_size = crtc->primary->fb->pitches[0];
 
         if (sdev->meram) {
                 /* Enable MERAM cache if configured. We need to de-init
@@ -402,7 +402,7 @@ static int shmob_drm_crtc_mode_set(struct drm_crtc *crtc,
                 }
 
                 cache = sh_mobile_meram_cache_alloc(sdev->meram, mdata,
-                                                    crtc->fb->pitches[0],
+                                                    crtc->primary->fb->pitches[0],
                                                     adjusted_mode->vdisplay,
                                                     format->meram,
                                                     &scrtc->line_size);
@@ -489,7 +489,7 @@ static int shmob_drm_crtc_page_flip(struct drm_crtc *crtc,
         }
         spin_unlock_irqrestore(&dev->event_lock, flags);
 
-        crtc->fb = fb;
+        crtc->primary->fb = fb;
         shmob_drm_crtc_update_base(scrtc);
 
         if (event) {
diff --git a/drivers/gpu/drm/tegra/Makefile b/drivers/gpu/drm/tegra/Makefile
index 8d220afbd85f..d43f21bb4596 100644
--- a/drivers/gpu/drm/tegra/Makefile
+++ b/drivers/gpu/drm/tegra/Makefile
@@ -11,6 +11,8 @@ tegra-drm-y := \
         hdmi.o \
         mipi-phy.o \
         dsi.o \
+        sor.o \
+        dpaux.o \
         gr2d.o \
         gr3d.o
 
diff --git a/drivers/gpu/drm/tegra/bus.c b/drivers/gpu/drm/tegra/bus.c
index e38e5967d77b..71cef5c13dc8 100644
--- a/drivers/gpu/drm/tegra/bus.c
+++ b/drivers/gpu/drm/tegra/bus.c
@@ -63,7 +63,7 @@ int drm_host1x_init(struct drm_driver *driver, struct host1x_device *device)
         return 0;
 
 err_free:
-        drm_dev_free(drm);
+        drm_dev_unref(drm);
         return ret;
 }
 
diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c
index 9336006b475d..36c717af6cf9 100644
--- a/drivers/gpu/drm/tegra/dc.c
+++ b/drivers/gpu/drm/tegra/dc.c
@@ -235,14 +235,14 @@ static void tegra_dc_finish_page_flip(struct tegra_dc *dc)
         if (!dc->event)
                 return;
 
-        bo = tegra_fb_get_plane(crtc->fb, 0);
+        bo = tegra_fb_get_plane(crtc->primary->fb, 0);
 
         /* check if new start address has been latched */
         tegra_dc_writel(dc, READ_MUX, DC_CMD_STATE_ACCESS);
         base = tegra_dc_readl(dc, DC_WINBUF_START_ADDR);
         tegra_dc_writel(dc, 0, DC_CMD_STATE_ACCESS);
 
-        if (base == bo->paddr + crtc->fb->offsets[0]) {
+        if (base == bo->paddr + crtc->primary->fb->offsets[0]) {
                 spin_lock_irqsave(&drm->event_lock, flags);
                 drm_send_vblank_event(drm, dc->pipe, dc->event);
                 drm_vblank_put(drm, dc->pipe);
@@ -284,7 +284,7 @@ static int tegra_dc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         }
 
         tegra_dc_set_base(dc, 0, 0, fb);
-        crtc->fb = fb;
+        crtc->primary->fb = fb;
 
         return 0;
 }
@@ -645,7 +645,7 @@ static int tegra_crtc_mode_set(struct drm_crtc *crtc,
                                struct drm_display_mode *adjusted,
                                int x, int y, struct drm_framebuffer *old_fb)
 {
-        struct tegra_bo *bo = tegra_fb_get_plane(crtc->fb, 0);
+        struct tegra_bo *bo = tegra_fb_get_plane(crtc->primary->fb, 0);
         struct tegra_dc *dc = to_tegra_dc(crtc);
         struct tegra_dc_window window;
         unsigned long div, value;
@@ -682,9 +682,9 @@ static int tegra_crtc_mode_set(struct drm_crtc *crtc,
         window.dst.y = 0;
         window.dst.w = mode->hdisplay;
         window.dst.h = mode->vdisplay;
-        window.format = tegra_dc_format(crtc->fb->pixel_format);
-        window.bits_per_pixel = crtc->fb->bits_per_pixel;
-        window.stride[0] = crtc->fb->pitches[0];
+        window.format = tegra_dc_format(crtc->primary->fb->pixel_format);
+        window.bits_per_pixel = crtc->primary->fb->bits_per_pixel;
+        window.stride[0] = crtc->primary->fb->pitches[0];
         window.base[0] = bo->paddr;
 
         err = tegra_dc_setup_window(dc, 0, &window);
@@ -699,7 +699,7 @@ static int tegra_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
 {
         struct tegra_dc *dc = to_tegra_dc(crtc);
 
-        return tegra_dc_set_base(dc, x, y, crtc->fb);
+        return tegra_dc_set_base(dc, x, y, crtc->primary->fb);
 }
 
 static void tegra_crtc_prepare(struct drm_crtc *crtc)
diff --git a/drivers/gpu/drm/tegra/dc.h b/drivers/gpu/drm/tegra/dc.h
index 3c2c0ea1cd87..c94101494826 100644
--- a/drivers/gpu/drm/tegra/dc.h
+++ b/drivers/gpu/drm/tegra/dc.h
@@ -118,6 +118,7 @@
 #define DC_DISP_DISP_WIN_OPTIONS                0x402
 #define HDMI_ENABLE (1 << 30)
 #define DSI_ENABLE  (1 << 29)
+#define SOR_ENABLE  (1 << 25)
 
 #define DC_DISP_DISP_MEM_HIGH_PRIORITY          0x403
 #define CURSOR_THRESHOLD(x)   (((x) & 0x03) << 24)
diff --git a/drivers/gpu/drm/tegra/dpaux.c b/drivers/gpu/drm/tegra/dpaux.c
new file mode 100644
index 000000000000..d536ed381fbd
--- /dev/null
+++ b/drivers/gpu/drm/tegra/dpaux.c
@@ -0,0 +1,544 @@
+/*
+ * Copyright (C) 2013 NVIDIA Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/gpio.h>
+#include <linux/interrupt.h>
+#include <linux/io.h>
+#include <linux/of_gpio.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/regulator/consumer.h>
+
+#include <drm/drm_dp_helper.h>
+#include <drm/drm_panel.h>
+
+#include "dpaux.h"
+#include "drm.h"
+
+static DEFINE_MUTEX(dpaux_lock);
+static LIST_HEAD(dpaux_list);
+
+struct tegra_dpaux {
+        struct drm_dp_aux aux;
+        struct device *dev;
+
+        void __iomem *regs;
+        int irq;
+
+        struct tegra_output *output;
+
+        struct reset_control *rst;
+        struct clk *clk_parent;
+        struct clk *clk;
+
+        struct regulator *vdd;
+
+        struct completion complete;
+        struct list_head list;
+};
+
+static inline struct tegra_dpaux *to_dpaux(struct drm_dp_aux *aux)
+{
+        return container_of(aux, struct tegra_dpaux, aux);
+}
+
+static inline unsigned long tegra_dpaux_readl(struct tegra_dpaux *dpaux,
+                                              unsigned long offset)
+{
+        return readl(dpaux->regs + (offset << 2));
+}
+
+static inline void tegra_dpaux_writel(struct tegra_dpaux *dpaux,
+                                      unsigned long value,
+                                      unsigned long offset)
+{
+        writel(value, dpaux->regs + (offset << 2));
+}
+
+static void tegra_dpaux_write_fifo(struct tegra_dpaux *dpaux, const u8 *buffer,
+                                   size_t size)
+{
+        unsigned long offset = DPAUX_DP_AUXDATA_WRITE(0);
+        size_t i, j;
+
+        for (i = 0; i < size; i += 4) {
+                size_t num = min_t(size_t, size - i, 4);
+                unsigned long value = 0;
+
+                for (j = 0; j < num; j++)
+                        value |= buffer[i + j] << (j * 8);
+
+                tegra_dpaux_writel(dpaux, value, offset++);
+        }
+}
+
+static void tegra_dpaux_read_fifo(struct tegra_dpaux *dpaux, u8 *buffer,
+                                  size_t size)
+{
+        unsigned long offset = DPAUX_DP_AUXDATA_READ(0);
+        size_t i, j;
+
+        for (i = 0; i < size; i += 4) {
+                size_t num = min_t(size_t, size - i, 4);
+                unsigned long value;
+
+                value = tegra_dpaux_readl(dpaux, offset++);
+
+                for (j = 0; j < num; j++)
+                        buffer[i + j] = value >> (j * 8);
+        }
+}
+
+static ssize_t tegra_dpaux_transfer(struct drm_dp_aux *aux,
+                                    struct drm_dp_aux_msg *msg)
+{
+        unsigned long value = DPAUX_DP_AUXCTL_TRANSACTREQ;
+        unsigned long timeout = msecs_to_jiffies(250);
+        struct tegra_dpaux *dpaux = to_dpaux(aux);
+        unsigned long status;
+        ssize_t ret = 0;
+
+        if (msg->size < 1 || msg->size > 16)
+                return -EINVAL;
+
+        tegra_dpaux_writel(dpaux, msg->address, DPAUX_DP_AUXADDR);
+
+        switch (msg->request & ~DP_AUX_I2C_MOT) {
+        case DP_AUX_I2C_WRITE:
+                if (msg->request & DP_AUX_I2C_MOT)
+                        value = DPAUX_DP_AUXCTL_CMD_MOT_WR;
+                else
+                        value = DPAUX_DP_AUXCTL_CMD_I2C_WR;
+
+                break;
+
+        case DP_AUX_I2C_READ:
+                if (msg->request & DP_AUX_I2C_MOT)
+                        value = DPAUX_DP_AUXCTL_CMD_MOT_RD;
+                else
+                        value = DPAUX_DP_AUXCTL_CMD_I2C_RD;
+
+                break;
+
+        case DP_AUX_I2C_STATUS:
+                if (msg->request & DP_AUX_I2C_MOT)
+                        value = DPAUX_DP_AUXCTL_CMD_MOT_RQ;
+                else
+                        value = DPAUX_DP_AUXCTL_CMD_I2C_RQ;
+
+                break;
+
+        case DP_AUX_NATIVE_WRITE:
+                value = DPAUX_DP_AUXCTL_CMD_AUX_WR;
+                break;
+
+        case DP_AUX_NATIVE_READ:
+                value = DPAUX_DP_AUXCTL_CMD_AUX_RD;
+                break;
+
+        default:
+                return -EINVAL;
+        }
+
+        value |= DPAUX_DP_AUXCTL_CMDLEN(msg->size - 1);
+        tegra_dpaux_writel(dpaux, value, DPAUX_DP_AUXCTL);
+
+        if ((msg->request & DP_AUX_I2C_READ) == 0) {
+                tegra_dpaux_write_fifo(dpaux, msg->buffer, msg->size);
+                ret = msg->size;
+        }
+
+        /* start transaction */
+        value = tegra_dpaux_readl(dpaux, DPAUX_DP_AUXCTL);
+        value |= DPAUX_DP_AUXCTL_TRANSACTREQ;
+        tegra_dpaux_writel(dpaux, value, DPAUX_DP_AUXCTL);
+
+        status = wait_for_completion_timeout(&dpaux->complete, timeout);
+        if (!status)
+                return -ETIMEDOUT;
+
+        /* read status and clear errors */
+        value = tegra_dpaux_readl(dpaux, DPAUX_DP_AUXSTAT);
+        tegra_dpaux_writel(dpaux, 0xf00, DPAUX_DP_AUXSTAT);
+
+        if (value & DPAUX_DP_AUXSTAT_TIMEOUT_ERROR)
+                return -ETIMEDOUT;
+
+        if ((value & DPAUX_DP_AUXSTAT_RX_ERROR) ||
+            (value & DPAUX_DP_AUXSTAT_SINKSTAT_ERROR) ||
+            (value & DPAUX_DP_AUXSTAT_NO_STOP_ERROR))
+                return -EIO;
+
+        switch ((value & DPAUX_DP_AUXSTAT_REPLY_TYPE_MASK) >> 16) {
+        case 0x00:
+                msg->reply = DP_AUX_NATIVE_REPLY_ACK;
+                break;
+
+        case 0x01:
+                msg->reply = DP_AUX_NATIVE_REPLY_NACK;
+                break;
+
+        case 0x02:
+                msg->reply = DP_AUX_NATIVE_REPLY_DEFER;
+                break;
+
+        case 0x04:
+                msg->reply = DP_AUX_I2C_REPLY_NACK;
+                break;
+
+        case 0x08:
+                msg->reply = DP_AUX_I2C_REPLY_DEFER;
+                break;
+        }
+
+        if (msg->reply == DP_AUX_NATIVE_REPLY_ACK) {
+                if (msg->request & DP_AUX_I2C_READ) {
+                        size_t count = value & DPAUX_DP_AUXSTAT_REPLY_MASK;
+
+                        if (WARN_ON(count != msg->size))
+                                count = min_t(size_t, count, msg->size);
+
+                        tegra_dpaux_read_fifo(dpaux, msg->buffer, count);
+                        ret = count;
+                }
+        }
+
+        return ret;
+}
+
+static irqreturn_t tegra_dpaux_irq(int irq, void *data)
+{
+        struct tegra_dpaux *dpaux = data;
+        irqreturn_t ret = IRQ_HANDLED;
+        unsigned long value;
+
+        /* clear interrupts */
+        value = tegra_dpaux_readl(dpaux, DPAUX_INTR_AUX);
+        tegra_dpaux_writel(dpaux, value, DPAUX_INTR_AUX);
+
+        if (value & DPAUX_INTR_PLUG_EVENT) {
+                if (dpaux->output) {
+                        drm_helper_hpd_irq_event(dpaux->output->connector.dev);
+                }
+        }
+
+        if (value & DPAUX_INTR_UNPLUG_EVENT) {
+                if (dpaux->output)
+                        drm_helper_hpd_irq_event(dpaux->output->connector.dev);
+        }
+
+        if (value & DPAUX_INTR_IRQ_EVENT) {
+                /* TODO: handle this */
+        }
+
+        if (value & DPAUX_INTR_AUX_DONE)
+                complete(&dpaux->complete);
+
+        return ret;
+}
+
+static int tegra_dpaux_probe(struct platform_device *pdev)
+{
+        struct tegra_dpaux *dpaux;
+        struct resource *regs;
+        unsigned long value;
+        int err;
+
+        dpaux = devm_kzalloc(&pdev->dev, sizeof(*dpaux), GFP_KERNEL);
+        if (!dpaux)
+                return -ENOMEM;
+
+        init_completion(&dpaux->complete);
+        INIT_LIST_HEAD(&dpaux->list);
+        dpaux->dev = &pdev->dev;
+
+        regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        dpaux->regs = devm_ioremap_resource(&pdev->dev, regs);
+        if (IS_ERR(dpaux->regs))
+                return PTR_ERR(dpaux->regs);
+
+        dpaux->irq = platform_get_irq(pdev, 0);
+        if (dpaux->irq < 0) {
+                dev_err(&pdev->dev, "failed to get IRQ\n");
+                return -ENXIO;
+        }
+
+        dpaux->rst = devm_reset_control_get(&pdev->dev, "dpaux");
+        if (IS_ERR(dpaux->rst))
+                return PTR_ERR(dpaux->rst);
+
+        dpaux->clk = devm_clk_get(&pdev->dev, NULL);
+        if (IS_ERR(dpaux->clk))
+                return PTR_ERR(dpaux->clk);
+
+        err = clk_prepare_enable(dpaux->clk);
+        if (err < 0)
+                return err;
+
+        reset_control_deassert(dpaux->rst);
+
+        dpaux->clk_parent = devm_clk_get(&pdev->dev, "parent");
+        if (IS_ERR(dpaux->clk_parent))
+                return PTR_ERR(dpaux->clk_parent);
+
+        err = clk_prepare_enable(dpaux->clk_parent);
+        if (err < 0)
+                return err;
+
+        err = clk_set_rate(dpaux->clk_parent, 270000000);
+        if (err < 0) {
+                dev_err(&pdev->dev, "failed to set clock to 270 MHz: %d\n",
+                        err);
+                return err;
+        }
+
+        dpaux->vdd = devm_regulator_get(&pdev->dev, "vdd");
+        if (IS_ERR(dpaux->vdd))
+                return PTR_ERR(dpaux->vdd);
+
+        err = devm_request_irq(dpaux->dev, dpaux->irq, tegra_dpaux_irq, 0,
+                               dev_name(dpaux->dev), dpaux);
+        if (err < 0) {
+                dev_err(dpaux->dev, "failed to request IRQ#%u: %d\n",
+                        dpaux->irq, err);
+                return err;
+        }
+
+        dpaux->aux.transfer = tegra_dpaux_transfer;
+        dpaux->aux.dev = &pdev->dev;
+
+        err = drm_dp_aux_register_i2c_bus(&dpaux->aux);
+        if (err < 0)
+                return err;
+
+        /* enable and clear all interrupts */
+        value = DPAUX_INTR_AUX_DONE | DPAUX_INTR_IRQ_EVENT |
+                DPAUX_INTR_UNPLUG_EVENT | DPAUX_INTR_PLUG_EVENT;
+        tegra_dpaux_writel(dpaux, value, DPAUX_INTR_EN_AUX);
+        tegra_dpaux_writel(dpaux, value, DPAUX_INTR_AUX);
+
+        mutex_lock(&dpaux_lock);
+        list_add_tail(&dpaux->list, &dpaux_list);
+        mutex_unlock(&dpaux_lock);
+
+        platform_set_drvdata(pdev, dpaux);
+
+        return 0;
+}
+
+static int tegra_dpaux_remove(struct platform_device *pdev)
+{
+        struct tegra_dpaux *dpaux = platform_get_drvdata(pdev);
+
+        drm_dp_aux_unregister_i2c_bus(&dpaux->aux);
+
+        mutex_lock(&dpaux_lock);
+        list_del(&dpaux->list);
+        mutex_unlock(&dpaux_lock);
+
+        clk_disable_unprepare(dpaux->clk_parent);
+        reset_control_assert(dpaux->rst);
+        clk_disable_unprepare(dpaux->clk);
+
+        return 0;
+}
+
+static const struct of_device_id tegra_dpaux_of_match[] = {
+        { .compatible = "nvidia,tegra124-dpaux", },
+        { },
+};
+
+struct platform_driver tegra_dpaux_driver = {
+        .driver = {
+                .name = "tegra-dpaux",
+                .of_match_table = tegra_dpaux_of_match,
+        },
+        .probe = tegra_dpaux_probe,
+        .remove = tegra_dpaux_remove,
+};
+
+struct tegra_dpaux *tegra_dpaux_find_by_of_node(struct device_node *np)
+{
+        struct tegra_dpaux *dpaux;
+
+        mutex_lock(&dpaux_lock);
+
+        list_for_each_entry(dpaux, &dpaux_list, list)
+                if (np == dpaux->dev->of_node) {
+                        mutex_unlock(&dpaux_lock);
+                        return dpaux;
+                }
+
+        mutex_unlock(&dpaux_lock);
+
+        return NULL;
+}
+
+int tegra_dpaux_attach(struct tegra_dpaux *dpaux, struct tegra_output *output)
+{
+        unsigned long timeout;
+        int err;
+
+        dpaux->output = output;
+
+        err = regulator_enable(dpaux->vdd);
+        if (err < 0)
+                return err;
+
+        timeout = jiffies + msecs_to_jiffies(250);
+
+        while (time_before(jiffies, timeout)) {
+                enum drm_connector_status status;
+
+                status = tegra_dpaux_detect(dpaux);
+                if (status == connector_status_connected)
+                        return 0;
+
+                usleep_range(1000, 2000);
+        }
+
+        return -ETIMEDOUT;
+}
+
+int tegra_dpaux_detach(struct tegra_dpaux *dpaux)
+{
+        unsigned long timeout;
+        int err;
+
+        err = regulator_disable(dpaux->vdd);
+        if (err < 0)
+                return err;
+
+        timeout = jiffies + msecs_to_jiffies(250);
+
+        while (time_before(jiffies, timeout)) {
+                enum drm_connector_status status;
+
+                status = tegra_dpaux_detect(dpaux);
+                if (status == connector_status_disconnected) {
+                        dpaux->output = NULL;
+                        return 0;
+                }
+
+                usleep_range(1000, 2000);
+        }
+
+        return -ETIMEDOUT;
+}
+
+enum drm_connector_status tegra_dpaux_detect(struct tegra_dpaux *dpaux)
+{
+        unsigned long value;
+
+        value = tegra_dpaux_readl(dpaux, DPAUX_DP_AUXSTAT);
+
+        if (value & DPAUX_DP_AUXSTAT_HPD_STATUS)
+                return connector_status_connected;
+
+        return connector_status_disconnected;
+}
+
+int tegra_dpaux_enable(struct tegra_dpaux *dpaux)
+{
+        unsigned long value;
+
+        value = DPAUX_HYBRID_PADCTL_AUX_CMH(2) |
+                DPAUX_HYBRID_PADCTL_AUX_DRVZ(4) |
+                DPAUX_HYBRID_PADCTL_AUX_DRVI(0x18) |
+                DPAUX_HYBRID_PADCTL_AUX_INPUT_RCV |
+                DPAUX_HYBRID_PADCTL_MODE_AUX;
+        tegra_dpaux_writel(dpaux, value, DPAUX_HYBRID_PADCTL);
+
+        value = tegra_dpaux_readl(dpaux, DPAUX_HYBRID_SPARE);
+        value &= ~DPAUX_HYBRID_SPARE_PAD_POWER_DOWN;
+        tegra_dpaux_writel(dpaux, value, DPAUX_HYBRID_SPARE);
+
+        return 0;
+}
+
+int tegra_dpaux_disable(struct tegra_dpaux *dpaux)
+{
+        unsigned long value;
+
+        value = tegra_dpaux_readl(dpaux, DPAUX_HYBRID_SPARE);
+        value |= DPAUX_HYBRID_SPARE_PAD_POWER_DOWN;
+        tegra_dpaux_writel(dpaux, value, DPAUX_HYBRID_SPARE);
+
+        return 0;
+}
+
+int tegra_dpaux_prepare(struct tegra_dpaux *dpaux, u8 encoding)
+{
+        int err;
+
+        err = drm_dp_dpcd_writeb(&dpaux->aux, DP_MAIN_LINK_CHANNEL_CODING_SET,
+                                 encoding);
+        if (err < 0)
+                return err;
+
+        return 0;
+}
+
+int tegra_dpaux_train(struct tegra_dpaux *dpaux, struct drm_dp_link *link,
+                      u8 pattern)
+{
+        u8 tp = pattern & DP_TRAINING_PATTERN_MASK;
+        u8 status[DP_LINK_STATUS_SIZE], values[4];
+        unsigned int i;
+        int err;
+
+        err = drm_dp_dpcd_writeb(&dpaux->aux, DP_TRAINING_PATTERN_SET, pattern);
+        if (err < 0)
+                return err;
+
+        if (tp == DP_TRAINING_PATTERN_DISABLE)
+                return 0;
+
+        for (i = 0; i < link->num_lanes; i++)
+                values[i] = DP_TRAIN_MAX_PRE_EMPHASIS_REACHED |
+                            DP_TRAIN_PRE_EMPHASIS_0 |
+                            DP_TRAIN_MAX_SWING_REACHED |
+                            DP_TRAIN_VOLTAGE_SWING_400;
+
+        err = drm_dp_dpcd_write(&dpaux->aux, DP_TRAINING_LANE0_SET, values,
+                                link->num_lanes);
+        if (err < 0)
+                return err;
+
+        usleep_range(500, 1000);
+
+        err = drm_dp_dpcd_read_link_status(&dpaux->aux, status);
+        if (err < 0)
+                return err;
+
+        switch (tp) {
+        case DP_TRAINING_PATTERN_1:
+                if (!drm_dp_clock_recovery_ok(status, link->num_lanes))
+                        return -EAGAIN;
+
+                break;
+
+        case DP_TRAINING_PATTERN_2:
+                if (!drm_dp_channel_eq_ok(status, link->num_lanes))
+                        return -EAGAIN;
+
+                break;
+
+        default:
+                dev_err(dpaux->dev, "unsupported training pattern %u\n", tp);
+                return -EINVAL;
+        }
+
+        err = drm_dp_dpcd_writeb(&dpaux->aux, DP_EDP_CONFIGURATION_SET, 0);
+        if (err < 0)
+                return err;
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/tegra/dpaux.h b/drivers/gpu/drm/tegra/dpaux.h
new file mode 100644
index 000000000000..4f5bf10fdff9
--- /dev/null
+++ b/drivers/gpu/drm/tegra/dpaux.h
@@ -0,0 +1,73 @@
+/*
+ * Copyright (C) 2013 NVIDIA Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef DRM_TEGRA_DPAUX_H
+#define DRM_TEGRA_DPAUX_H
+
+#define DPAUX_CTXSW 0x00
+
+#define DPAUX_INTR_EN_AUX 0x01
+#define DPAUX_INTR_AUX 0x05
+#define DPAUX_INTR_AUX_DONE (1 << 3)
+#define DPAUX_INTR_IRQ_EVENT (1 << 2)
+#define DPAUX_INTR_UNPLUG_EVENT (1 << 1)
+#define DPAUX_INTR_PLUG_EVENT (1 << 0)
+
+#define DPAUX_DP_AUXDATA_WRITE(x) (0x09 + ((x) << 2))
+#define DPAUX_DP_AUXDATA_READ(x) (0x19 + ((x) << 2))
+#define DPAUX_DP_AUXADDR 0x29
+
+#define DPAUX_DP_AUXCTL 0x2d
+#define DPAUX_DP_AUXCTL_TRANSACTREQ (1 << 16)
+#define DPAUX_DP_AUXCTL_CMD_AUX_RD (9 << 12)
+#define DPAUX_DP_AUXCTL_CMD_AUX_WR (8 << 12)
+#define DPAUX_DP_AUXCTL_CMD_MOT_RQ (6 << 12)
+#define DPAUX_DP_AUXCTL_CMD_MOT_RD (5 << 12)
+#define DPAUX_DP_AUXCTL_CMD_MOT_WR (4 << 12)
+#define DPAUX_DP_AUXCTL_CMD_I2C_RQ (2 << 12)
+#define DPAUX_DP_AUXCTL_CMD_I2C_RD (1 << 12)
+#define DPAUX_DP_AUXCTL_CMD_I2C_WR (0 << 12)
+#define DPAUX_DP_AUXCTL_CMDLEN(x) ((x) & 0xff)
+
+#define DPAUX_DP_AUXSTAT 0x31
+#define DPAUX_DP_AUXSTAT_HPD_STATUS (1 << 28)
+#define DPAUX_DP_AUXSTAT_REPLY_TYPE_MASK (0xf0000)
+#define DPAUX_DP_AUXSTAT_NO_STOP_ERROR (1 << 11)
+#define DPAUX_DP_AUXSTAT_SINKSTAT_ERROR (1 << 10)
+#define DPAUX_DP_AUXSTAT_RX_ERROR (1 << 9)
+#define DPAUX_DP_AUXSTAT_TIMEOUT_ERROR (1 << 8)
+#define DPAUX_DP_AUXSTAT_REPLY_MASK (0xff)
+
+#define DPAUX_DP_AUX_SINKSTAT_LO 0x35
+#define DPAUX_DP_AUX_SINKSTAT_HI 0x39
+
+#define DPAUX_HPD_CONFIG 0x3d
+#define DPAUX_HPD_CONFIG_UNPLUG_MIN_TIME(x) (((x) & 0xffff) << 16)
+#define DPAUX_HPD_CONFIG_PLUG_MIN_TIME(x) ((x) & 0xffff)
+
+#define DPAUX_HPD_IRQ_CONFIG 0x41
+#define DPAUX_HPD_IRQ_CONFIG_MIN_LOW_TIME(x) ((x) & 0xffff)
+
+#define DPAUX_DP_AUX_CONFIG 0x45
+
+#define DPAUX_HYBRID_PADCTL 0x49
+#define DPAUX_HYBRID_PADCTL_AUX_CMH(x) (((x) & 0x3) << 12)
+#define DPAUX_HYBRID_PADCTL_AUX_DRVZ(x) (((x) & 0x7) << 8)
+#define DPAUX_HYBRID_PADCTL_AUX_DRVI(x) (((x) & 0x3f) << 2)
+#define DPAUX_HYBRID_PADCTL_AUX_INPUT_RCV (1 << 1)
+#define DPAUX_HYBRID_PADCTL_MODE_I2C (1 << 0)
+#define DPAUX_HYBRID_PADCTL_MODE_AUX (0 << 0)
+
+#define DPAUX_HYBRID_SPARE 0x4d
+#define DPAUX_HYBRID_SPARE_PAD_POWER_DOWN (1 << 0)
+
+#define DPAUX_SCRATCH_REG0 0x51
+#define DPAUX_SCRATCH_REG1 0x55
+#define DPAUX_SCRATCH_REG2 0x59
+
+#endif
diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
index c71594754f46..6f5b6e2f552e 100644
--- a/drivers/gpu/drm/tegra/drm.c
+++ b/drivers/gpu/drm/tegra/drm.c
@@ -665,6 +665,7 @@ static const struct of_device_id host1x_drm_subdevs[] = {
         { .compatible = "nvidia,tegra114-hdmi", },
         { .compatible = "nvidia,tegra114-gr3d", },
         { .compatible = "nvidia,tegra124-dc", },
+        { .compatible = "nvidia,tegra124-sor", },
         { /* sentinel */ }
 };
 
@@ -691,14 +692,22 @@ static int __init host1x_drm_init(void)
         if (err < 0)
                 goto unregister_dc;
 
-        err = platform_driver_register(&tegra_hdmi_driver);
+        err = platform_driver_register(&tegra_sor_driver);
         if (err < 0)
                 goto unregister_dsi;
 
-        err = platform_driver_register(&tegra_gr2d_driver);
+        err = platform_driver_register(&tegra_hdmi_driver);
+        if (err < 0)
+                goto unregister_sor;
+
+        err = platform_driver_register(&tegra_dpaux_driver);
         if (err < 0)
                 goto unregister_hdmi;
 
+        err = platform_driver_register(&tegra_gr2d_driver);
+        if (err < 0)
+                goto unregister_dpaux;
+
         err = platform_driver_register(&tegra_gr3d_driver);
         if (err < 0)
                 goto unregister_gr2d;
@@ -707,8 +716,12 @@ static int __init host1x_drm_init(void)
 
 unregister_gr2d:
         platform_driver_unregister(&tegra_gr2d_driver);
+unregister_dpaux:
+        platform_driver_unregister(&tegra_dpaux_driver);
 unregister_hdmi:
         platform_driver_unregister(&tegra_hdmi_driver);
+unregister_sor:
+        platform_driver_unregister(&tegra_sor_driver);
 unregister_dsi:
         platform_driver_unregister(&tegra_dsi_driver);
 unregister_dc:
@@ -723,7 +736,9 @@ static void __exit host1x_drm_exit(void)
 {
         platform_driver_unregister(&tegra_gr3d_driver);
         platform_driver_unregister(&tegra_gr2d_driver);
+        platform_driver_unregister(&tegra_dpaux_driver);
         platform_driver_unregister(&tegra_hdmi_driver);
+        platform_driver_unregister(&tegra_sor_driver);
         platform_driver_unregister(&tegra_dsi_driver);
         platform_driver_unregister(&tegra_dc_driver);
         host1x_driver_unregister(&host1x_drm_driver);
diff --git a/drivers/gpu/drm/tegra/drm.h b/drivers/gpu/drm/tegra/drm.h
index bf1cac7658f8..126332c3ecbb 100644
--- a/drivers/gpu/drm/tegra/drm.h
+++ b/drivers/gpu/drm/tegra/drm.h
@@ -179,12 +179,14 @@ struct tegra_output_ops {
         int (*check_mode)(struct tegra_output *output,
                           struct drm_display_mode *mode,
                           enum drm_mode_status *status);
+        enum drm_connector_status (*detect)(struct tegra_output *output);
 };
 
 enum tegra_output_type {
         TEGRA_OUTPUT_RGB,
         TEGRA_OUTPUT_HDMI,
         TEGRA_OUTPUT_DSI,
+        TEGRA_OUTPUT_EDP,
 };
 
 struct tegra_output {
@@ -265,6 +267,22 @@ extern int tegra_output_remove(struct tegra_output *output);
 extern int tegra_output_init(struct drm_device *drm, struct tegra_output *output);
 extern int tegra_output_exit(struct tegra_output *output);
 
+/* from dpaux.c */
+
+struct tegra_dpaux;
+struct drm_dp_link;
+struct drm_dp_aux;
+
+struct tegra_dpaux *tegra_dpaux_find_by_of_node(struct device_node *np);
+enum drm_connector_status tegra_dpaux_detect(struct tegra_dpaux *dpaux);
+int tegra_dpaux_attach(struct tegra_dpaux *dpaux, struct tegra_output *output);
+int tegra_dpaux_detach(struct tegra_dpaux *dpaux);
+int tegra_dpaux_enable(struct tegra_dpaux *dpaux);
+int tegra_dpaux_disable(struct tegra_dpaux *dpaux);
+int tegra_dpaux_prepare(struct tegra_dpaux *dpaux, u8 encoding);
+int tegra_dpaux_train(struct tegra_dpaux *dpaux, struct drm_dp_link *link,
+                      u8 pattern);
+
 /* from fb.c */
 struct tegra_bo *tegra_fb_get_plane(struct drm_framebuffer *framebuffer,
                                     unsigned int index);
@@ -278,7 +296,9 @@ extern void tegra_fbdev_restore_mode(struct tegra_fbdev *fbdev);
 
 extern struct platform_driver tegra_dc_driver;
 extern struct platform_driver tegra_dsi_driver;
+extern struct platform_driver tegra_sor_driver;
 extern struct platform_driver tegra_hdmi_driver;
+extern struct platform_driver tegra_dpaux_driver;
 extern struct platform_driver tegra_gr2d_driver;
 extern struct platform_driver tegra_gr3d_driver;
 
diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c
index d452faab0235..0e599f0417c0 100644
--- a/drivers/gpu/drm/tegra/dsi.c
+++ b/drivers/gpu/drm/tegra/dsi.c
@@ -1,23 +1,9 @@
 /*
  * Copyright (C) 2013 NVIDIA Corporation
  *
- * Permission to use, copy, modify, distribute, and sell this software and its
- * documentation for any purpose is hereby granted without fee, provided that
- * the above copyright notice appear in all copies and that both that copyright
- * notice and this permission notice appear in supporting documentation, and
- * that the name of the copyright holders not be used in advertising or
- * publicity pertaining to distribution of the software without specific,
- * written prior permission.  The copyright holders make no representations
- * about the suitability of this software for any purpose.  It is provided "as
- * is" without express or implied warranty.
- *
- * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
- * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
- * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
- * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
- * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
- * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
- * OF THIS SOFTWARE.
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
  */
 
 #include <linux/clk.h>
diff --git a/drivers/gpu/drm/tegra/dsi.h b/drivers/gpu/drm/tegra/dsi.h
index 00e79c1f448c..1db5cc24ea91 100644
--- a/drivers/gpu/drm/tegra/dsi.h
+++ b/drivers/gpu/drm/tegra/dsi.h
@@ -1,23 +1,9 @@
 /*
  * Copyright (C) 2013 NVIDIA Corporation
  *
- * Permission to use, copy, modify, distribute, and sell this software and its
- * documentation for any purpose is hereby granted without fee, provided that
- * the above copyright notice appear in all copies and that both that copyright
- * notice and this permission notice appear in supporting documentation, and
- * that the name of the copyright holders not be used in advertising or
- * publicity pertaining to distribution of the software without specific,
- * written prior permission.  The copyright holders make no representations
- * about the suitability of this software for any purpose.  It is provided "as
- * is" without express or implied warranty.
- *
- * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
- * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
- * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
- * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
- * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
- * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
- * OF THIS SOFTWARE.
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
  */
 
 #ifndef DRM_TEGRA_DSI_H
diff --git a/drivers/gpu/drm/tegra/gem.c b/drivers/gpu/drm/tegra/gem.c
index ef853e558036..bcf9895cef9f 100644
--- a/drivers/gpu/drm/tegra/gem.c
+++ b/drivers/gpu/drm/tegra/gem.c
@@ -8,14 +8,9 @@
  *
  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
  *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version 2
- * of the License, or (at your option) any later version.
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
  */
 
 #include <linux/dma-buf.h>
@@ -394,6 +389,18 @@ static int tegra_gem_prime_mmap(struct dma_buf *buf, struct vm_area_struct *vma)
         return -EINVAL;
 }
 
+static void *tegra_gem_prime_vmap(struct dma_buf *buf)
+{
+        struct drm_gem_object *gem = buf->priv;
+        struct tegra_bo *bo = to_tegra_bo(gem);
+
+        return bo->vaddr;
+}
+
+static void tegra_gem_prime_vunmap(struct dma_buf *buf, void *vaddr)
+{
+}
+
 static const struct dma_buf_ops tegra_gem_prime_dmabuf_ops = {
         .map_dma_buf = tegra_gem_prime_map_dma_buf,
         .unmap_dma_buf = tegra_gem_prime_unmap_dma_buf,
@@ -403,6 +410,8 @@ static const struct dma_buf_ops tegra_gem_prime_dmabuf_ops = {
         .kmap = tegra_gem_prime_kmap,
         .kunmap = tegra_gem_prime_kunmap,
         .mmap = tegra_gem_prime_mmap,
+        .vmap = tegra_gem_prime_vmap,
+        .vunmap = tegra_gem_prime_vunmap,
 };
 
 struct dma_buf *tegra_gem_prime_export(struct drm_device *drm,
diff --git a/drivers/gpu/drm/tegra/gem.h b/drivers/gpu/drm/tegra/gem.h
index ffd4f792b410..2f3fe96c5154 100644
--- a/drivers/gpu/drm/tegra/gem.h
+++ b/drivers/gpu/drm/tegra/gem.h
@@ -3,17 +3,9 @@
  *
  * Copyright (c) 2012-2013, NVIDIA Corporation.
  *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
  */
 
 #ifndef __HOST1X_GEM_H
diff --git a/drivers/gpu/drm/tegra/gr2d.c b/drivers/gpu/drm/tegra/gr2d.c
index 7ec4259ffded..2c7ca748edf5 100644
--- a/drivers/gpu/drm/tegra/gr2d.c
+++ b/drivers/gpu/drm/tegra/gr2d.c
@@ -1,17 +1,9 @@
 /*
  * Copyright (c) 2012-2013, NVIDIA Corporation.
  *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms and conditions of the GNU General Public License,
- * version 2, as published by the Free Software Foundation.
- *
- * This program is distributed in the hope it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
- * more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
  */
 
 #include <linux/clk.h>
diff --git a/drivers/gpu/drm/tegra/mipi-phy.c b/drivers/gpu/drm/tegra/mipi-phy.c
index e2c4aedaee78..486d19d589c8 100644
--- a/drivers/gpu/drm/tegra/mipi-phy.c
+++ b/drivers/gpu/drm/tegra/mipi-phy.c
@@ -1,23 +1,9 @@
 /*
  * Copyright (C) 2013 NVIDIA Corporation
  *
- * Permission to use, copy, modify, distribute, and sell this software and its
- * documentation for any purpose is hereby granted without fee, provided that
- * the above copyright notice appear in all copies and that both that copyright
- * notice and this permission notice appear in supporting documentation, and
- * that the name of the copyright holders not be used in advertising or
- * publicity pertaining to distribution of the software without specific,
- * written prior permission.  The copyright holders make no representations
- * about the suitability of this software for any purpose.  It is provided "as
- * is" without express or implied warranty.
- *
- * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
- * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
- * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
- * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
- * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
- * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
- * OF THIS SOFTWARE.
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
  */
 
 #include <linux/errno.h>
diff --git a/drivers/gpu/drm/tegra/mipi-phy.h b/drivers/gpu/drm/tegra/mipi-phy.h
index d3591694432d..012ea8ac36d7 100644
--- a/drivers/gpu/drm/tegra/mipi-phy.h
+++ b/drivers/gpu/drm/tegra/mipi-phy.h
@@ -1,23 +1,9 @@
 /*
  * Copyright (C) 2013 NVIDIA Corporation
  *
- * Permission to use, copy, modify, distribute, and sell this software and its
- * documentation for any purpose is hereby granted without fee, provided that
- * the above copyright notice appear in all copies and that both that copyright
- * notice and this permission notice appear in supporting documentation, and
- * that the name of the copyright holders not be used in advertising or
- * publicity pertaining to distribution of the software without specific,
- * written prior permission.  The copyright holders make no representations
- * about the suitability of this software for any purpose.  It is provided "as
- * is" without express or implied warranty.
- *
- * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
- * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
- * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
- * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
- * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
- * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
- * OF THIS SOFTWARE.
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
  */
 
 #ifndef DRM_TEGRA_MIPI_PHY_H
diff --git a/drivers/gpu/drm/tegra/output.c b/drivers/gpu/drm/tegra/output.c
index 57cecbd18ca8..a3e4f1eca6f7 100644
--- a/drivers/gpu/drm/tegra/output.c
+++ b/drivers/gpu/drm/tegra/output.c
@@ -77,6 +77,9 @@ tegra_connector_detect(struct drm_connector *connector, bool force)
         struct tegra_output *output = connector_to_output(connector);
         enum drm_connector_status status = connector_status_unknown;
 
+        if (output->ops->detect)
+                return output->ops->detect(output);
+
         if (gpio_is_valid(output->hpd_gpio)) {
                 if (gpio_get_value(output->hpd_gpio) == 0)
                         status = connector_status_disconnected;
@@ -292,6 +295,11 @@ int tegra_output_init(struct drm_device *drm, struct tegra_output *output)
                 encoder = DRM_MODE_ENCODER_DSI;
                 break;
 
+        case TEGRA_OUTPUT_EDP:
+                connector = DRM_MODE_CONNECTOR_eDP;
+                encoder = DRM_MODE_ENCODER_TMDS;
+                break;
+
         default:
                 connector = DRM_MODE_CONNECTOR_Unknown;
                 encoder = DRM_MODE_ENCODER_NONE;
diff --git a/drivers/gpu/drm/tegra/sor.c b/drivers/gpu/drm/tegra/sor.c
new file mode 100644
index 000000000000..49ef5729f435
--- /dev/null
+++ b/drivers/gpu/drm/tegra/sor.c
@@ -0,0 +1,1092 @@
+/*
+ * Copyright (C) 2013 NVIDIA Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <linux/reset.h>
+#include <linux/tegra-powergate.h>
+
+#include <drm/drm_dp_helper.h>
+
+#include "dc.h"
+#include "drm.h"
+#include "sor.h"
+
+struct tegra_sor {
+        struct host1x_client client;
+        struct tegra_output output;
+        struct device *dev;
+
+        void __iomem *regs;
+
+        struct reset_control *rst;
+        struct clk *clk_parent;
+        struct clk *clk_safe;
+        struct clk *clk_dp;
+        struct clk *clk;
+
+        struct tegra_dpaux *dpaux;
+
+        bool enabled;
+};
+
+static inline struct tegra_sor *
+host1x_client_to_sor(struct host1x_client *client)
+{
+        return container_of(client, struct tegra_sor, client);
+}
+
+static inline struct tegra_sor *to_sor(struct tegra_output *output)
+{
+        return container_of(output, struct tegra_sor, output);
+}
+
+static inline unsigned long tegra_sor_readl(struct tegra_sor *sor,
+                                            unsigned long offset)
+{
+        return readl(sor->regs + (offset << 2));
+}
+
+static inline void tegra_sor_writel(struct tegra_sor *sor, unsigned long value,
+                                    unsigned long offset)
+{
+        writel(value, sor->regs + (offset << 2));
+}
+
+static int tegra_sor_dp_train_fast(struct tegra_sor *sor,
+                                   struct drm_dp_link *link)
+{
+        unsigned long value;
+        unsigned int i;
+        u8 pattern;
+        int err;
+
+        /* setup lane parameters */
+        value = SOR_LANE_DRIVE_CURRENT_LANE3(0x40) |
+                SOR_LANE_DRIVE_CURRENT_LANE2(0x40) |
+                SOR_LANE_DRIVE_CURRENT_LANE1(0x40) |
+                SOR_LANE_DRIVE_CURRENT_LANE0(0x40);
+        tegra_sor_writel(sor, value, SOR_LANE_DRIVE_CURRENT_0);
+
+        value = SOR_LANE_PREEMPHASIS_LANE3(0x0f) |
+                SOR_LANE_PREEMPHASIS_LANE2(0x0f) |
+                SOR_LANE_PREEMPHASIS_LANE1(0x0f) |
+                SOR_LANE_PREEMPHASIS_LANE0(0x0f);
+        tegra_sor_writel(sor, value, SOR_LANE_PREEMPHASIS_0);
+
+        value = SOR_LANE_POST_CURSOR_LANE3(0x00) |
+                SOR_LANE_POST_CURSOR_LANE2(0x00) |
+                SOR_LANE_POST_CURSOR_LANE1(0x00) |
+                SOR_LANE_POST_CURSOR_LANE0(0x00);
+        tegra_sor_writel(sor, value, SOR_LANE_POST_CURSOR_0);
+
+        /* disable LVDS mode */
+        tegra_sor_writel(sor, 0, SOR_LVDS);
+
+        value = tegra_sor_readl(sor, SOR_DP_PADCTL_0);
+        value |= SOR_DP_PADCTL_TX_PU_ENABLE;
+        value &= ~SOR_DP_PADCTL_TX_PU_MASK;
+        value |= SOR_DP_PADCTL_TX_PU(2); /* XXX: don't hardcode? */
+        tegra_sor_writel(sor, value, SOR_DP_PADCTL_0);
+
+        value = tegra_sor_readl(sor, SOR_DP_PADCTL_0);
+        value |= SOR_DP_PADCTL_CM_TXD_3 | SOR_DP_PADCTL_CM_TXD_2 |
+                 SOR_DP_PADCTL_CM_TXD_1 | SOR_DP_PADCTL_CM_TXD_0;
+        tegra_sor_writel(sor, value, SOR_DP_PADCTL_0);
+
+        usleep_range(10, 100);
+
+        value = tegra_sor_readl(sor, SOR_DP_PADCTL_0);
+        value &= ~(SOR_DP_PADCTL_CM_TXD_3 | SOR_DP_PADCTL_CM_TXD_2 |
+                   SOR_DP_PADCTL_CM_TXD_1 | SOR_DP_PADCTL_CM_TXD_0);
+        tegra_sor_writel(sor, value, SOR_DP_PADCTL_0);
+
+        err = tegra_dpaux_prepare(sor->dpaux, DP_SET_ANSI_8B10B);
+        if (err < 0)
+                return err;
+
+        for (i = 0, value = 0; i < link->num_lanes; i++) {
+                unsigned long lane = SOR_DP_TPG_CHANNEL_CODING |
+                                     SOR_DP_TPG_SCRAMBLER_NONE |
+                                     SOR_DP_TPG_PATTERN_TRAIN1;
+                value = (value << 8) | lane;
+        }
+
+        tegra_sor_writel(sor, value, SOR_DP_TPG);
+
+        pattern = DP_TRAINING_PATTERN_1;
+
+        err = tegra_dpaux_train(sor->dpaux, link, pattern);
+        if (err < 0)
+                return err;
+
+        value = tegra_sor_readl(sor, SOR_DP_SPARE_0);
+        value |= SOR_DP_SPARE_SEQ_ENABLE;
+        value &= ~SOR_DP_SPARE_PANEL_INTERNAL;
+        value |= SOR_DP_SPARE_MACRO_SOR_CLK;
+        tegra_sor_writel(sor, value, SOR_DP_SPARE_0);
+
+        for (i = 0, value = 0; i < link->num_lanes; i++) {
+                unsigned long lane = SOR_DP_TPG_CHANNEL_CODING |
+                                     SOR_DP_TPG_SCRAMBLER_NONE |
+                                     SOR_DP_TPG_PATTERN_TRAIN2;
+                value = (value << 8) | lane;
+        }
+
+        tegra_sor_writel(sor, value, SOR_DP_TPG);
+
+        pattern = DP_LINK_SCRAMBLING_DISABLE | DP_TRAINING_PATTERN_2;
+
+        err = tegra_dpaux_train(sor->dpaux, link, pattern);
+        if (err < 0)
+                return err;
+
+        for (i = 0, value = 0; i < link->num_lanes; i++) {
+                unsigned long lane = SOR_DP_TPG_CHANNEL_CODING |
+                                     SOR_DP_TPG_SCRAMBLER_GALIOS |
+                                     SOR_DP_TPG_PATTERN_NONE;
+                value = (value << 8) | lane;
+        }
+
+        tegra_sor_writel(sor, value, SOR_DP_TPG);
+
+        pattern = DP_TRAINING_PATTERN_DISABLE;
+
+        err = tegra_dpaux_train(sor->dpaux, link, pattern);
+        if (err < 0)
+                return err;
+
+        return 0;
+}
+
+static void tegra_sor_super_update(struct tegra_sor *sor)
+{
+        tegra_sor_writel(sor, 0, SOR_SUPER_STATE_0);
+        tegra_sor_writel(sor, 1, SOR_SUPER_STATE_0);
+        tegra_sor_writel(sor, 0, SOR_SUPER_STATE_0);
+}
+
+static void tegra_sor_update(struct tegra_sor *sor)
+{
+        tegra_sor_writel(sor, 0, SOR_STATE_0);
+        tegra_sor_writel(sor, 1, SOR_STATE_0);
+        tegra_sor_writel(sor, 0, SOR_STATE_0);
+}
+
+static int tegra_sor_setup_pwm(struct tegra_sor *sor, unsigned long timeout)
+{
+        unsigned long value;
+
+        value = tegra_sor_readl(sor, SOR_PWM_DIV);
+        value &= ~SOR_PWM_DIV_MASK;
+        value |= 0x400; /* period */
+        tegra_sor_writel(sor, value, SOR_PWM_DIV);
+
+        value = tegra_sor_readl(sor, SOR_PWM_CTL);
+        value &= ~SOR_PWM_CTL_DUTY_CYCLE_MASK;
+        value |= 0x400; /* duty cycle */
+        value &= ~SOR_PWM_CTL_CLK_SEL; /* clock source: PCLK */
+        value |= SOR_PWM_CTL_TRIGGER;
+        tegra_sor_writel(sor, value, SOR_PWM_CTL);
+
+        timeout = jiffies + msecs_to_jiffies(timeout);
+
+        while (time_before(jiffies, timeout)) {
+                value = tegra_sor_readl(sor, SOR_PWM_CTL);
+                if ((value & SOR_PWM_CTL_TRIGGER) == 0)
+                        return 0;
+
+                usleep_range(25, 100);
+        }
+
+        return -ETIMEDOUT;
+}
+
+static int tegra_sor_attach(struct tegra_sor *sor)
+{
+        unsigned long value, timeout;
+
+        /* wake up in normal mode */
+        value = tegra_sor_readl(sor, SOR_SUPER_STATE_1);
+        value |= SOR_SUPER_STATE_HEAD_MODE_AWAKE;
+        value |= SOR_SUPER_STATE_MODE_NORMAL;
+        tegra_sor_writel(sor, value, SOR_SUPER_STATE_1);
+        tegra_sor_super_update(sor);
+
+        /* attach */
+        value = tegra_sor_readl(sor, SOR_SUPER_STATE_1);
+        value |= SOR_SUPER_STATE_ATTACHED;
+        tegra_sor_writel(sor, value, SOR_SUPER_STATE_1);
+        tegra_sor_super_update(sor);
+
+        timeout = jiffies + msecs_to_jiffies(250);
+
+        while (time_before(jiffies, timeout)) {
+                value = tegra_sor_readl(sor, SOR_TEST);
+                if ((value & SOR_TEST_ATTACHED) != 0)
+                        return 0;
+
+                usleep_range(25, 100);
+        }
+
+        return -ETIMEDOUT;
+}
+
+static int tegra_sor_wakeup(struct tegra_sor *sor)
+{
+        struct tegra_dc *dc = to_tegra_dc(sor->output.encoder.crtc);
+        unsigned long value, timeout;
+
+        /* enable display controller outputs */
+        value = tegra_dc_readl(dc, DC_CMD_DISPLAY_POWER_CONTROL);
+        value |= PW0_ENABLE | PW1_ENABLE | PW2_ENABLE | PW3_ENABLE |
+                 PW4_ENABLE | PM0_ENABLE | PM1_ENABLE;
+        tegra_dc_writel(dc, value, DC_CMD_DISPLAY_POWER_CONTROL);
+
+        tegra_dc_writel(dc, GENERAL_ACT_REQ << 8, DC_CMD_STATE_CONTROL);
+        tegra_dc_writel(dc, GENERAL_ACT_REQ, DC_CMD_STATE_CONTROL);
+
+        timeout = jiffies + msecs_to_jiffies(250);
+
+        /* wait for head to wake up */
+        while (time_before(jiffies, timeout)) {
+                value = tegra_sor_readl(sor, SOR_TEST);
+                value &= SOR_TEST_HEAD_MODE_MASK;
+
+                if (value == SOR_TEST_HEAD_MODE_AWAKE)
+                        return 0;
+
+                usleep_range(25, 100);
+        }
+
+        return -ETIMEDOUT;
+}
+
+static int tegra_sor_power_up(struct tegra_sor *sor, unsigned long timeout)
+{
+        unsigned long value;
+
+        value = tegra_sor_readl(sor, SOR_PWR);
+        value |= SOR_PWR_TRIGGER | SOR_PWR_NORMAL_STATE_PU;
+        tegra_sor_writel(sor, value, SOR_PWR);
+
+        timeout = jiffies + msecs_to_jiffies(timeout);
+
+        while (time_before(jiffies, timeout)) {
+                value = tegra_sor_readl(sor, SOR_PWR);
+                if ((value & SOR_PWR_TRIGGER) == 0)
+                        return 0;
+
+                usleep_range(25, 100);
+        }
+
+        return -ETIMEDOUT;
+}
+
+static int tegra_output_sor_enable(struct tegra_output *output)
+{
+        struct tegra_dc *dc = to_tegra_dc(output->encoder.crtc);
+        struct drm_display_mode *mode = &dc->base.mode;
+        unsigned int vbe, vse, hbe, hse, vbs, hbs, i;
+        struct tegra_sor *sor = to_sor(output);
+        unsigned long value;
+        int err;
+
+        if (sor->enabled)
+                return 0;
+
+        err = clk_prepare_enable(sor->clk);
+        if (err < 0)
+                return err;
+
+        reset_control_deassert(sor->rst);
+
+        if (sor->dpaux) {
+                err = tegra_dpaux_enable(sor->dpaux);
+                if (err < 0)
+                        dev_err(sor->dev, "failed to enable DP: %d\n", err);
+        }
+
+        err = clk_set_parent(sor->clk, sor->clk_safe);
+        if (err < 0)
+                dev_err(sor->dev, "failed to set safe parent clock: %d\n", err);
+
+        value = tegra_sor_readl(sor, SOR_CLK_CNTRL);
+        value &= ~SOR_CLK_CNTRL_DP_CLK_SEL_MASK;
+        value |= SOR_CLK_CNTRL_DP_CLK_SEL_SINGLE_DPCLK;
+        tegra_sor_writel(sor, value, SOR_CLK_CNTRL);
+
+        value = tegra_sor_readl(sor, SOR_PLL_2);
+        value &= ~SOR_PLL_2_BANDGAP_POWERDOWN;
+        tegra_sor_writel(sor, value, SOR_PLL_2);
+        usleep_range(20, 100);
+
+        value = tegra_sor_readl(sor, SOR_PLL_3);
+        value |= SOR_PLL_3_PLL_VDD_MODE_V3_3;
+        tegra_sor_writel(sor, value, SOR_PLL_3);
+
+        value = SOR_PLL_0_ICHPMP(0xf) | SOR_PLL_0_VCOCAP_RST |
+                SOR_PLL_0_PLLREG_LEVEL_V45 | SOR_PLL_0_RESISTOR_EXT;
+        tegra_sor_writel(sor, value, SOR_PLL_0);
+
+        value = tegra_sor_readl(sor, SOR_PLL_2);
+        value |= SOR_PLL_2_SEQ_PLLCAPPD;
+        value &= ~SOR_PLL_2_SEQ_PLLCAPPD_ENFORCE;
+        value |= SOR_PLL_2_LVDS_ENABLE;
+        tegra_sor_writel(sor, value, SOR_PLL_2);
+
+        value = SOR_PLL_1_TERM_COMPOUT | SOR_PLL_1_TMDS_TERM;
+        tegra_sor_writel(sor, value, SOR_PLL_1);
+
+        while (true) {
+                value = tegra_sor_readl(sor, SOR_PLL_2);
+                if ((value & SOR_PLL_2_SEQ_PLLCAPPD_ENFORCE) == 0)
+                        break;
+
+                usleep_range(250, 1000);
+        }
+
+        value = tegra_sor_readl(sor, SOR_PLL_2);
+        value &= ~SOR_PLL_2_POWERDOWN_OVERRIDE;
+        value &= ~SOR_PLL_2_PORT_POWERDOWN;
+        tegra_sor_writel(sor, value, SOR_PLL_2);
+
+        /*
+         * power up
+         */
+
+        /* set safe link bandwidth (1.62 Gbps) */
+        value = tegra_sor_readl(sor, SOR_CLK_CNTRL);
+        value &= ~SOR_CLK_CNTRL_DP_LINK_SPEED_MASK;
+        value |= SOR_CLK_CNTRL_DP_LINK_SPEED_G1_62;
+        tegra_sor_writel(sor, value, SOR_CLK_CNTRL);
+
+        /* step 1 */
+        value = tegra_sor_readl(sor, SOR_PLL_2);
+        value |= SOR_PLL_2_SEQ_PLLCAPPD_ENFORCE | SOR_PLL_2_PORT_POWERDOWN |
+                 SOR_PLL_2_BANDGAP_POWERDOWN;
+        tegra_sor_writel(sor, value, SOR_PLL_2);
+
+        value = tegra_sor_readl(sor, SOR_PLL_0);
+        value |= SOR_PLL_0_VCOPD | SOR_PLL_0_POWER_OFF;
+        tegra_sor_writel(sor, value, SOR_PLL_0);
+
+        value = tegra_sor_readl(sor, SOR_DP_PADCTL_0);
+        value &= ~SOR_DP_PADCTL_PAD_CAL_PD;
+        tegra_sor_writel(sor, value, SOR_DP_PADCTL_0);
+
+        /* step 2 */
+        err = tegra_io_rail_power_on(TEGRA_IO_RAIL_LVDS);
+        if (err < 0) {
+                dev_err(sor->dev, "failed to power on I/O rail: %d\n", err);
+                return err;
+        }
+
+        usleep_range(5, 100);
+
+        /* step 3 */
+        value = tegra_sor_readl(sor, SOR_PLL_2);
+        value &= ~SOR_PLL_2_BANDGAP_POWERDOWN;
+        tegra_sor_writel(sor, value, SOR_PLL_2);
+
+        usleep_range(20, 100);
+
+        /* step 4 */
+        value = tegra_sor_readl(sor, SOR_PLL_0);
+        value &= ~SOR_PLL_0_POWER_OFF;
+        value &= ~SOR_PLL_0_VCOPD;
+        tegra_sor_writel(sor, value, SOR_PLL_0);
+
+        value = tegra_sor_readl(sor, SOR_PLL_2);
+        value &= ~SOR_PLL_2_SEQ_PLLCAPPD_ENFORCE;
+        tegra_sor_writel(sor, value, SOR_PLL_2);
+
+        usleep_range(200, 1000);
+
+        /* step 5 */
+        value = tegra_sor_readl(sor, SOR_PLL_2);
+        value &= ~SOR_PLL_2_PORT_POWERDOWN;
+        tegra_sor_writel(sor, value, SOR_PLL_2);
+
+        /* switch to DP clock */
+        err = clk_set_parent(sor->clk, sor->clk_dp);
+        if (err < 0)
+                dev_err(sor->dev, "failed to set DP parent clock: %d\n", err);
+
+        /* power dplanes (XXX parameterize based on link?) */
+        value = tegra_sor_readl(sor, SOR_DP_PADCTL_0);
+        value |= SOR_DP_PADCTL_PD_TXD_3 | SOR_DP_PADCTL_PD_TXD_0 |
+                 SOR_DP_PADCTL_PD_TXD_1 | SOR_DP_PADCTL_PD_TXD_2;
+        tegra_sor_writel(sor, value, SOR_DP_PADCTL_0);
+
+        value = tegra_sor_readl(sor, SOR_DP_LINKCTL_0);
+        value &= ~SOR_DP_LINKCTL_LANE_COUNT_MASK;
+        value |= SOR_DP_LINKCTL_LANE_COUNT(4);
+        tegra_sor_writel(sor, value, SOR_DP_LINKCTL_0);
+
+        /* start lane sequencer */
+        value = SOR_LANE_SEQ_CTL_TRIGGER | SOR_LANE_SEQ_CTL_SEQUENCE_DOWN |
+                SOR_LANE_SEQ_CTL_POWER_STATE_UP;
+        tegra_sor_writel(sor, value, SOR_LANE_SEQ_CTL);
+
+        while (true) {
+                value = tegra_sor_readl(sor, SOR_LANE_SEQ_CTL);
+                if ((value & SOR_LANE_SEQ_CTL_TRIGGER) == 0)
+                        break;
+
+                usleep_range(250, 1000);
+        }
+
+        /* set link bandwidth (2.7 GHz, XXX: parameterize based on link?) */
+        value = tegra_sor_readl(sor, SOR_CLK_CNTRL);
+        value &= ~SOR_CLK_CNTRL_DP_LINK_SPEED_MASK;
+        value |= SOR_CLK_CNTRL_DP_LINK_SPEED_G2_70;
+        tegra_sor_writel(sor, value, SOR_CLK_CNTRL);
+
+        /* set linkctl */
+        value = tegra_sor_readl(sor, SOR_DP_LINKCTL_0);
+        value |= SOR_DP_LINKCTL_ENABLE;
+
+        value &= ~SOR_DP_LINKCTL_TU_SIZE_MASK;
+        value |= SOR_DP_LINKCTL_TU_SIZE(59); /* XXX: don't hardcode? */
+
+        value |= SOR_DP_LINKCTL_ENHANCED_FRAME;
+        tegra_sor_writel(sor, value, SOR_DP_LINKCTL_0);
+
+        for (i = 0, value = 0; i < 4; i++) {
+                unsigned long lane = SOR_DP_TPG_CHANNEL_CODING |
+                                     SOR_DP_TPG_SCRAMBLER_GALIOS |
+                                     SOR_DP_TPG_PATTERN_NONE;
+                value = (value << 8) | lane;
+        }
+
+        tegra_sor_writel(sor, value, SOR_DP_TPG);
+
+        value = tegra_sor_readl(sor, SOR_DP_CONFIG_0);
+        value &= ~SOR_DP_CONFIG_WATERMARK_MASK;
+        value |= SOR_DP_CONFIG_WATERMARK(14); /* XXX: don't hardcode? */
+
+        value &= ~SOR_DP_CONFIG_ACTIVE_SYM_COUNT_MASK;
+        value |= SOR_DP_CONFIG_ACTIVE_SYM_COUNT(47); /* XXX: don't hardcode? */
+
+        value &= ~SOR_DP_CONFIG_ACTIVE_SYM_FRAC_MASK;
+        value |= SOR_DP_CONFIG_ACTIVE_SYM_FRAC(9); /* XXX: don't hardcode? */
+
+        value &= ~SOR_DP_CONFIG_ACTIVE_SYM_POLARITY; /* XXX: don't hardcode? */
+
+        value |= SOR_DP_CONFIG_ACTIVE_SYM_ENABLE;
+        value |= SOR_DP_CONFIG_DISPARITY_NEGATIVE; /* XXX: don't hardcode? */
+        tegra_sor_writel(sor, value, SOR_DP_CONFIG_0);
+
+        value = tegra_sor_readl(sor, SOR_DP_AUDIO_HBLANK_SYMBOLS);
+        value &= ~SOR_DP_AUDIO_HBLANK_SYMBOLS_MASK;
+        value |= 137; /* XXX: don't hardcode? */
+        tegra_sor_writel(sor, value, SOR_DP_AUDIO_HBLANK_SYMBOLS);
+
+        value = tegra_sor_readl(sor, SOR_DP_AUDIO_VBLANK_SYMBOLS);
+        value &= ~SOR_DP_AUDIO_VBLANK_SYMBOLS_MASK;
+        value |= 2368; /* XXX: don't hardcode? */
+        tegra_sor_writel(sor, value, SOR_DP_AUDIO_VBLANK_SYMBOLS);
+
+        /* enable pad calibration logic */
+        value = tegra_sor_readl(sor, SOR_DP_PADCTL_0);
+        value |= SOR_DP_PADCTL_PAD_CAL_PD;
+        tegra_sor_writel(sor, value, SOR_DP_PADCTL_0);
+
+        if (sor->dpaux) {
+                /* FIXME: properly convert to struct drm_dp_aux */
+                struct drm_dp_aux *aux = (struct drm_dp_aux *)sor->dpaux;
+                struct drm_dp_link link;
+                u8 rate, lanes;
+
+                err = drm_dp_link_probe(aux, &link);
+                if (err < 0) {
+                        dev_err(sor->dev, "failed to probe eDP link: %d\n",
+                                err);
+                        return err;
+                }
+
+                err = drm_dp_link_power_up(aux, &link);
+                if (err < 0) {
+                        dev_err(sor->dev, "failed to power up eDP link: %d\n",
+                                err);
+                        return err;
+                }
+
+                err = drm_dp_link_configure(aux, &link);
+                if (err < 0) {
+                        dev_err(sor->dev, "failed to configure eDP link: %d\n",
+                                err);
+                        return err;
+                }
+
+                rate = drm_dp_link_rate_to_bw_code(link.rate);
+                lanes = link.num_lanes;
+
+                value = tegra_sor_readl(sor, SOR_CLK_CNTRL);
+                value &= ~SOR_CLK_CNTRL_DP_LINK_SPEED_MASK;
+                value |= SOR_CLK_CNTRL_DP_LINK_SPEED(rate);
+                tegra_sor_writel(sor, value, SOR_CLK_CNTRL);
+
+                value = tegra_sor_readl(sor, SOR_DP_LINKCTL_0);
+                value &= ~SOR_DP_LINKCTL_LANE_COUNT_MASK;
+                value |= SOR_DP_LINKCTL_LANE_COUNT(lanes);
+
+                if (link.capabilities & DP_LINK_CAP_ENHANCED_FRAMING)
+                        value |= SOR_DP_LINKCTL_ENHANCED_FRAME;
+
+                tegra_sor_writel(sor, value, SOR_DP_LINKCTL_0);
+
+                /* disable training pattern generator */
+
+                for (i = 0; i < link.num_lanes; i++) {
+                        unsigned long lane = SOR_DP_TPG_CHANNEL_CODING |
+                                             SOR_DP_TPG_SCRAMBLER_GALIOS |
+                                             SOR_DP_TPG_PATTERN_NONE;
+                        value = (value << 8) | lane;
+                }
+
+                tegra_sor_writel(sor, value, SOR_DP_TPG);
+
+                err = tegra_sor_dp_train_fast(sor, &link);
+                if (err < 0) {
+                        dev_err(sor->dev, "DP fast link training failed: %d\n",
+                                err);
+                        return err;
+                }
+
+                dev_dbg(sor->dev, "fast link training succeeded\n");
+        }
+
+        err = tegra_sor_power_up(sor, 250);
+        if (err < 0) {
+                dev_err(sor->dev, "failed to power up SOR: %d\n", err);
+                return err;
+        }
+
+        /* start display controller in continuous mode */
+        value = tegra_dc_readl(dc, DC_CMD_STATE_ACCESS);
+        value |= WRITE_MUX;
+        tegra_dc_writel(dc, value, DC_CMD_STATE_ACCESS);
+
+        tegra_dc_writel(dc, VSYNC_H_POSITION(1), DC_DISP_DISP_TIMING_OPTIONS);
+        tegra_dc_writel(dc, DISP_CTRL_MODE_C_DISPLAY, DC_CMD_DISPLAY_COMMAND);
+
+        value = tegra_dc_readl(dc, DC_CMD_STATE_ACCESS);
+        value &= ~WRITE_MUX;
+        tegra_dc_writel(dc, value, DC_CMD_STATE_ACCESS);
+
+        /*
+         * configure panel (24bpp, vsync-, hsync-, DP-A protocol, complete
+         * raster, associate with display controller)
+         */
+        value = SOR_STATE_ASY_PIXELDEPTH_BPP_24_444 |
+                SOR_STATE_ASY_VSYNCPOL |
+                SOR_STATE_ASY_HSYNCPOL |
+                SOR_STATE_ASY_PROTOCOL_DP_A |
+                SOR_STATE_ASY_CRC_MODE_COMPLETE |
+                SOR_STATE_ASY_OWNER(dc->pipe + 1);
+        tegra_sor_writel(sor, value, SOR_STATE_1);
+
+        /*
+         * TODO: The video timing programming below doesn't seem to match the
+         * register definitions.
+         */
+
+        value = ((mode->vtotal & 0x7fff) << 16) | (mode->htotal & 0x7fff);
+        tegra_sor_writel(sor, value, SOR_HEAD_STATE_1(0));
+
+        vse = mode->vsync_end - mode->vsync_start - 1;
+        hse = mode->hsync_end - mode->hsync_start - 1;
+
+        value = ((vse & 0x7fff) << 16) | (hse & 0x7fff);
+        tegra_sor_writel(sor, value, SOR_HEAD_STATE_2(0));
+
+        vbe = vse + (mode->vsync_start - mode->vdisplay);
+        hbe = hse + (mode->hsync_start - mode->hdisplay);
+
+        value = ((vbe & 0x7fff) << 16) | (hbe & 0x7fff);
+        tegra_sor_writel(sor, value, SOR_HEAD_STATE_3(0));
+
+        vbs = vbe + mode->vdisplay;
+        hbs = hbe + mode->hdisplay;
+
+        value = ((vbs & 0x7fff) << 16) | (hbs & 0x7fff);
+        tegra_sor_writel(sor, value, SOR_HEAD_STATE_4(0));
+
+        /* XXX interlaced mode */
+        tegra_sor_writel(sor, 0x00000001, SOR_HEAD_STATE_5(0));
+
+        /* CSTM (LVDS, link A/B, upper) */
+        value = SOR_CSTM_LVDS | SOR_CSTM_LINK_ACT_B | SOR_CSTM_LINK_ACT_B |
+                SOR_CSTM_UPPER;
+        tegra_sor_writel(sor, value, SOR_CSTM);
+
+        /* PWM setup */
+        err = tegra_sor_setup_pwm(sor, 250);
+        if (err < 0) {
+                dev_err(sor->dev, "failed to setup PWM: %d\n", err);
+                return err;
+        }
+
+        value = tegra_dc_readl(dc, DC_DISP_DISP_WIN_OPTIONS);
+        value |= SOR_ENABLE;
+        tegra_dc_writel(dc, value, DC_DISP_DISP_WIN_OPTIONS);
+
+        tegra_sor_update(sor);
+
+        err = tegra_sor_attach(sor);
+        if (err < 0) {
+                dev_err(sor->dev, "failed to attach SOR: %d\n", err);
+                return err;
+        }
+
+        err = tegra_sor_wakeup(sor);
+        if (err < 0) {
+                dev_err(sor->dev, "failed to enable DC: %d\n", err);
+                return err;
+        }
+
+        sor->enabled = true;
+
+        return 0;
+}
+
+static int tegra_sor_detach(struct tegra_sor *sor)
+{
+        unsigned long value, timeout;
+
+        /* switch to safe mode */
+        value = tegra_sor_readl(sor, SOR_SUPER_STATE_1);
+        value &= ~SOR_SUPER_STATE_MODE_NORMAL;
+        tegra_sor_writel(sor, value, SOR_SUPER_STATE_1);
+        tegra_sor_super_update(sor);
+
+        timeout = jiffies + msecs_to_jiffies(250);
+
+        while (time_before(jiffies, timeout)) {
+                value = tegra_sor_readl(sor, SOR_PWR);
+                if (value & SOR_PWR_MODE_SAFE)
+                        break;
+        }
+
+        if ((value & SOR_PWR_MODE_SAFE) == 0)
+                return -ETIMEDOUT;
+
+        /* go to sleep */
+        value = tegra_sor_readl(sor, SOR_SUPER_STATE_1);
+        value &= ~SOR_SUPER_STATE_HEAD_MODE_MASK;
+        tegra_sor_writel(sor, value, SOR_SUPER_STATE_1);
+        tegra_sor_super_update(sor);
+
+        /* detach */
+        value = tegra_sor_readl(sor, SOR_SUPER_STATE_1);
+        value &= ~SOR_SUPER_STATE_ATTACHED;
+        tegra_sor_writel(sor, value, SOR_SUPER_STATE_1);
+        tegra_sor_super_update(sor);
+
+        timeout = jiffies + msecs_to_jiffies(250);
+
+        while (time_before(jiffies, timeout)) {
+                value = tegra_sor_readl(sor, SOR_TEST);
+                if ((value & SOR_TEST_ATTACHED) == 0)
+                        break;
+
+                usleep_range(25, 100);
+        }
+
+        if ((value & SOR_TEST_ATTACHED) != 0)
+                return -ETIMEDOUT;
+
+        return 0;
+}
+
+static int tegra_sor_power_down(struct tegra_sor *sor)
+{
+        unsigned long value, timeout;
+        int err;
+
+        value = tegra_sor_readl(sor, SOR_PWR);
+        value &= ~SOR_PWR_NORMAL_STATE_PU;
+        value |= SOR_PWR_TRIGGER;
+        tegra_sor_writel(sor, value, SOR_PWR);
+
+        timeout = jiffies + msecs_to_jiffies(250);
+
+        while (time_before(jiffies, timeout)) {
+                value = tegra_sor_readl(sor, SOR_PWR);
+                if ((value & SOR_PWR_TRIGGER) == 0)
+                        return 0;
+
+                usleep_range(25, 100);
+        }
+
+        if ((value & SOR_PWR_TRIGGER) != 0)
+                return -ETIMEDOUT;
+
+        err = clk_set_parent(sor->clk, sor->clk_safe);
+        if (err < 0)
+                dev_err(sor->dev, "failed to set safe parent clock: %d\n", err);
+
+        value = tegra_sor_readl(sor, SOR_DP_PADCTL_0);
+        value &= ~(SOR_DP_PADCTL_PD_TXD_3 | SOR_DP_PADCTL_PD_TXD_0 |
+                   SOR_DP_PADCTL_PD_TXD_1 | SOR_DP_PADCTL_PD_TXD_2);
+        tegra_sor_writel(sor, value, SOR_DP_PADCTL_0);
+
+        /* stop lane sequencer */
+        value = SOR_LANE_SEQ_CTL_TRIGGER | SOR_LANE_SEQ_CTL_SEQUENCE_DOWN |
+                SOR_LANE_SEQ_CTL_POWER_STATE_DOWN;
+        tegra_sor_writel(sor, value, SOR_LANE_SEQ_CTL);
+
+        timeout = jiffies + msecs_to_jiffies(250);
+
+        while (time_before(jiffies, timeout)) {
+                value = tegra_sor_readl(sor, SOR_LANE_SEQ_CTL);
+                if ((value & SOR_LANE_SEQ_CTL_TRIGGER) == 0)
+                        break;
+
+                usleep_range(25, 100);
+        }
+
+        if ((value & SOR_LANE_SEQ_CTL_TRIGGER) != 0)
+                return -ETIMEDOUT;
+
+        value = tegra_sor_readl(sor, SOR_PLL_2);
+        value |= SOR_PLL_2_PORT_POWERDOWN;
+        tegra_sor_writel(sor, value, SOR_PLL_2);
+
+        usleep_range(20, 100);
+
+        value = tegra_sor_readl(sor, SOR_PLL_0);
+        value |= SOR_PLL_0_POWER_OFF;
+        value |= SOR_PLL_0_VCOPD;
+        tegra_sor_writel(sor, value, SOR_PLL_0);
+
+        value = tegra_sor_readl(sor, SOR_PLL_2);
+        value |= SOR_PLL_2_SEQ_PLLCAPPD;
+        value |= SOR_PLL_2_SEQ_PLLCAPPD_ENFORCE;
+        tegra_sor_writel(sor, value, SOR_PLL_2);
+
+        usleep_range(20, 100);
+
+        return 0;
+}
+
+static int tegra_output_sor_disable(struct tegra_output *output)
+{
+        struct tegra_dc *dc = to_tegra_dc(output->encoder.crtc);
+        struct tegra_sor *sor = to_sor(output);
+        unsigned long value;
+        int err;
+
+        if (!sor->enabled)
+                return 0;
+
+        err = tegra_sor_detach(sor);
+        if (err < 0) {
+                dev_err(sor->dev, "failed to detach SOR: %d\n", err);
+                return err;
+        }
+
+        tegra_sor_writel(sor, 0, SOR_STATE_1);
+        tegra_sor_update(sor);
+
+        /*
+         * The following accesses registers of the display controller, so make
+         * sure it's only executed when the output is attached to one.
+         */
+        if (dc) {
+                /*
+                 * XXX: We can't do this here because it causes the SOR to go
+                 * into an erroneous state and the output will look scrambled
+                 * the next time it is enabled. Presumably this is because we
+                 * should be doing this only on the next VBLANK. A possible
+                 * solution would be to queue a "power-off" event to trigger
+                 * this code to be run during the next VBLANK.
+                 */
+                /*
+                value = tegra_dc_readl(dc, DC_CMD_DISPLAY_POWER_CONTROL);
+                value &= ~(PW0_ENABLE | PW1_ENABLE | PW2_ENABLE | PW3_ENABLE |
+                           PW4_ENABLE | PM0_ENABLE | PM1_ENABLE);
+                tegra_dc_writel(dc, value, DC_CMD_DISPLAY_POWER_CONTROL);
+                */
+
+                value = tegra_dc_readl(dc, DC_CMD_DISPLAY_COMMAND);
+                value &= ~DISP_CTRL_MODE_MASK;
+                tegra_dc_writel(dc, value, DC_CMD_DISPLAY_COMMAND);
+
+                value = tegra_dc_readl(dc, DC_DISP_DISP_WIN_OPTIONS);
+                value &= ~SOR_ENABLE;
+                tegra_dc_writel(dc, value, DC_DISP_DISP_WIN_OPTIONS);
+
+                tegra_dc_writel(dc, GENERAL_ACT_REQ << 8, DC_CMD_STATE_CONTROL);
+                tegra_dc_writel(dc, GENERAL_ACT_REQ, DC_CMD_STATE_CONTROL);
+        }
+
+        err = tegra_sor_power_down(sor);
+        if (err < 0) {
+                dev_err(sor->dev, "failed to power down SOR: %d\n", err);
+                return err;
+        }
+
+        if (sor->dpaux) {
+                err = tegra_dpaux_disable(sor->dpaux);
+                if (err < 0) {
+                        dev_err(sor->dev, "failed to disable DP: %d\n", err);
+                        return err;
+                }
+        }
+
+        err = tegra_io_rail_power_off(TEGRA_IO_RAIL_LVDS);
+        if (err < 0) {
+                dev_err(sor->dev, "failed to power off I/O rail: %d\n", err);
+                return err;
+        }
+
+        reset_control_assert(sor->rst);
+        clk_disable_unprepare(sor->clk);
+
+        sor->enabled = false;
+
+        return 0;
+}
+
+static int tegra_output_sor_setup_clock(struct tegra_output *output,
+                                        struct clk *clk, unsigned long pclk)
+{
+        struct tegra_sor *sor = to_sor(output);
+        int err;
+
+        /* round to next MHz */
+        pclk = DIV_ROUND_UP(pclk / 2, 1000000) * 1000000;
+
+        err = clk_set_parent(clk, sor->clk_parent);
+        if (err < 0) {
+                dev_err(sor->dev, "failed to set parent clock: %d\n", err);
+                return err;
+        }
+
+        err = clk_set_rate(sor->clk_parent, pclk);
+        if (err < 0) {
+                dev_err(sor->dev, "failed to set base clock rate to %lu Hz\n",
+                        pclk * 2);
+                return err;
+        }
+
+        return 0;
+}
+
+static int tegra_output_sor_check_mode(struct tegra_output *output,
+                                       struct drm_display_mode *mode,
+                                       enum drm_mode_status *status)
+{
+        /*
+         * FIXME: For now, always assume that the mode is okay.
+         */
+
+        *status = MODE_OK;
+
+        return 0;
+}
+
+static enum drm_connector_status
+tegra_output_sor_detect(struct tegra_output *output)
+{
+        struct tegra_sor *sor = to_sor(output);
+
+        if (sor->dpaux)
+                return tegra_dpaux_detect(sor->dpaux);
+
+        return connector_status_unknown;
+}
+
+static const struct tegra_output_ops sor_ops = {
+        .enable = tegra_output_sor_enable,
+        .disable = tegra_output_sor_disable,
+        .setup_clock = tegra_output_sor_setup_clock,
+        .check_mode = tegra_output_sor_check_mode,
+        .detect = tegra_output_sor_detect,
+};
+
+static int tegra_sor_init(struct host1x_client *client)
+{
+        struct tegra_drm *tegra = dev_get_drvdata(client->parent);
+        struct tegra_sor *sor = host1x_client_to_sor(client);
+        int err;
+
+        if (!sor->dpaux)
+                return -ENODEV;
+
+        sor->output.type = TEGRA_OUTPUT_EDP;
+
+        sor->output.dev = sor->dev;
+        sor->output.ops = &sor_ops;
+
+        err = tegra_output_init(tegra->drm, &sor->output);
+        if (err < 0) {
+                dev_err(sor->dev, "output setup failed: %d\n", err);
+                return err;
+        }
+
+        if (sor->dpaux) {
+                err = tegra_dpaux_attach(sor->dpaux, &sor->output);
+                if (err < 0) {
+                        dev_err(sor->dev, "failed to attach DP: %d\n", err);
+                        return err;
+                }
+        }
+
+        return 0;
+}
+
+static int tegra_sor_exit(struct host1x_client *client)
+{
+        struct tegra_sor *sor = host1x_client_to_sor(client);
+        int err;
+
+        err = tegra_output_disable(&sor->output);
+        if (err < 0) {
+                dev_err(sor->dev, "output failed to disable: %d\n", err);
+                return err;
+        }
+
+        if (sor->dpaux) {
+                err = tegra_dpaux_detach(sor->dpaux);
+                if (err < 0) {
+                        dev_err(sor->dev, "failed to detach DP: %d\n", err);
+                        return err;
+                }
+        }
+
+        err = tegra_output_exit(&sor->output);
+        if (err < 0) {
+                dev_err(sor->dev, "output cleanup failed: %d\n", err);
+                return err;
+        }
+
+        return 0;
+}
+
+static const struct host1x_client_ops sor_client_ops = {
+        .init = tegra_sor_init,
+        .exit = tegra_sor_exit,
+};
+
+static int tegra_sor_probe(struct platform_device *pdev)
+{
+        struct device_node *np;
+        struct tegra_sor *sor;
+        struct resource *regs;
+        int err;
+
+        sor = devm_kzalloc(&pdev->dev, sizeof(*sor), GFP_KERNEL);
+        if (!sor)
+                return -ENOMEM;
+
+        sor->output.dev = sor->dev = &pdev->dev;
+
+        np = of_parse_phandle(pdev->dev.of_node, "nvidia,dpaux", 0);
+        if (np) {
+                sor->dpaux = tegra_dpaux_find_by_of_node(np);
+                of_node_put(np);
+
+                if (!sor->dpaux)
+                        return -EPROBE_DEFER;
+        }
+
+        err = tegra_output_probe(&sor->output);
+        if (err < 0)
+                return err;
+
+        regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        sor->regs = devm_ioremap_resource(&pdev->dev, regs);
+        if (IS_ERR(sor->regs))
+                return PTR_ERR(sor->regs);
+
+        sor->rst = devm_reset_control_get(&pdev->dev, "sor");
+        if (IS_ERR(sor->rst))
+                return PTR_ERR(sor->rst);
+
+        sor->clk = devm_clk_get(&pdev->dev, NULL);
+        if (IS_ERR(sor->clk))
+                return PTR_ERR(sor->clk);
+
+        sor->clk_parent = devm_clk_get(&pdev->dev, "parent");
+        if (IS_ERR(sor->clk_parent))
+                return PTR_ERR(sor->clk_parent);
+
+        err = clk_prepare_enable(sor->clk_parent);
+        if (err < 0)
+                return err;
+
+        sor->clk_safe = devm_clk_get(&pdev->dev, "safe");
+        if (IS_ERR(sor->clk_safe))
+                return PTR_ERR(sor->clk_safe);
+
+        err = clk_prepare_enable(sor->clk_safe);
+        if (err < 0)
+                return err;
+
+        sor->clk_dp = devm_clk_get(&pdev->dev, "dp");
+        if (IS_ERR(sor->clk_dp))
+                return PTR_ERR(sor->clk_dp);
+
+        err = clk_prepare_enable(sor->clk_dp);
+        if (err < 0)
+                return err;
+
+        INIT_LIST_HEAD(&sor->client.list);
+        sor->client.ops = &sor_client_ops;
+        sor->client.dev = &pdev->dev;
+
+        err = host1x_client_register(&sor->client);
+        if (err < 0) {
+                dev_err(&pdev->dev, "failed to register host1x client: %d\n",
+                        err);
+                return err;
+        }
+
+        platform_set_drvdata(pdev, sor);
+
+        return 0;
+}
+
+static int tegra_sor_remove(struct platform_device *pdev)
+{
+        struct tegra_sor *sor = platform_get_drvdata(pdev);
+        int err;
+
+        err = host1x_client_unregister(&sor->client);
+        if (err < 0) {
+                dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
+                        err);
+                return err;
+        }
+
+        clk_disable_unprepare(sor->clk_parent);
+        clk_disable_unprepare(sor->clk_safe);
+        clk_disable_unprepare(sor->clk_dp);
+        clk_disable_unprepare(sor->clk);
+
+        return 0;
+}
+
+static const struct of_device_id tegra_sor_of_match[] = {
+        { .compatible = "nvidia,tegra124-sor", },
+        { },
+};
+
+struct platform_driver tegra_sor_driver = {
+        .driver = {
+                .name = "tegra-sor",
+                .of_match_table = tegra_sor_of_match,
+        },
+        .probe = tegra_sor_probe,
+        .remove = tegra_sor_remove,
+};
diff --git a/drivers/gpu/drm/tegra/sor.h b/drivers/gpu/drm/tegra/sor.h
new file mode 100644
index 000000000000..f4156d54cd05
--- /dev/null
+++ b/drivers/gpu/drm/tegra/sor.h
@@ -0,0 +1,278 @@
+/*
+ * Copyright (C) 2013 NVIDIA Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef DRM_TEGRA_SOR_H
+#define DRM_TEGRA_SOR_H
+
+#define SOR_CTXSW 0x00
+
+#define SOR_SUPER_STATE_0 0x01
+
+#define SOR_SUPER_STATE_1 0x02
+#define  SOR_SUPER_STATE_ATTACHED               (1 << 3)
+#define  SOR_SUPER_STATE_MODE_NORMAL            (1 << 2)
+#define  SOR_SUPER_STATE_HEAD_MODE_MASK         (3 << 0)
+#define  SOR_SUPER_STATE_HEAD_MODE_AWAKE        (2 << 0)
+#define  SOR_SUPER_STATE_HEAD_MODE_SNOOZE       (1 << 0)
+#define  SOR_SUPER_STATE_HEAD_MODE_SLEEP        (0 << 0)
+
+#define SOR_STATE_0 0x03
+
+#define SOR_STATE_1 0x04
+#define  SOR_STATE_ASY_PIXELDEPTH_MASK          (0xf << 17)
+#define  SOR_STATE_ASY_PIXELDEPTH_BPP_18_444    (0x2 << 17)
+#define  SOR_STATE_ASY_PIXELDEPTH_BPP_24_444    (0x5 << 17)
+#define  SOR_STATE_ASY_VSYNCPOL                 (1 << 13)
+#define  SOR_STATE_ASY_HSYNCPOL                 (1 << 12)
+#define  SOR_STATE_ASY_PROTOCOL_MASK            (0xf << 8)
+#define  SOR_STATE_ASY_PROTOCOL_CUSTOM          (0xf << 8)
+#define  SOR_STATE_ASY_PROTOCOL_DP_A            (0x8 << 8)
+#define  SOR_STATE_ASY_PROTOCOL_DP_B            (0x9 << 8)
+#define  SOR_STATE_ASY_PROTOCOL_LVDS            (0x0 << 8)
+#define  SOR_STATE_ASY_CRC_MODE_MASK            (0x3 << 6)
+#define  SOR_STATE_ASY_CRC_MODE_NON_ACTIVE      (0x2 << 6)
+#define  SOR_STATE_ASY_CRC_MODE_COMPLETE        (0x1 << 6)
+#define  SOR_STATE_ASY_CRC_MODE_ACTIVE          (0x0 << 6)
+#define  SOR_STATE_ASY_OWNER(x)                 (((x) & 0xf) << 0)
+
+#define SOR_HEAD_STATE_0(x) (0x05 + (x))
+#define SOR_HEAD_STATE_1(x) (0x07 + (x))
+#define SOR_HEAD_STATE_2(x) (0x09 + (x))
+#define SOR_HEAD_STATE_3(x) (0x0b + (x))
+#define SOR_HEAD_STATE_4(x) (0x0d + (x))
+#define SOR_HEAD_STATE_5(x) (0x0f + (x))
+#define SOR_CRC_CNTRL 0x11
+#define SOR_DP_DEBUG_MVID 0x12
+
+#define SOR_CLK_CNTRL 0x13
+#define  SOR_CLK_CNTRL_DP_LINK_SPEED_MASK       (0x1f << 2)
+#define  SOR_CLK_CNTRL_DP_LINK_SPEED(x)         (((x) & 0x1f) << 2)
+#define  SOR_CLK_CNTRL_DP_LINK_SPEED_G1_62      (0x06 << 2)
+#define  SOR_CLK_CNTRL_DP_LINK_SPEED_G2_70      (0x0a << 2)
+#define  SOR_CLK_CNTRL_DP_LINK_SPEED_G5_40      (0x14 << 2)
+#define  SOR_CLK_CNTRL_DP_CLK_SEL_MASK          (3 << 0)
+#define  SOR_CLK_CNTRL_DP_CLK_SEL_SINGLE_PCLK   (0 << 0)
+#define  SOR_CLK_CNTRL_DP_CLK_SEL_DIFF_PCLK     (1 << 0)
+#define  SOR_CLK_CNTRL_DP_CLK_SEL_SINGLE_DPCLK  (2 << 0)
+#define  SOR_CLK_CNTRL_DP_CLK_SEL_DIFF_DPCLK    (3 << 0)
+
+#define SOR_CAP 0x14
+
+#define SOR_PWR 0x15
+#define  SOR_PWR_TRIGGER                        (1 << 31)
+#define  SOR_PWR_MODE_SAFE                      (1 << 28)
+#define  SOR_PWR_NORMAL_STATE_PU                (1 << 0)
+
+#define SOR_TEST 0x16
+#define  SOR_TEST_ATTACHED                      (1 << 10)
+#define  SOR_TEST_HEAD_MODE_MASK                (3 << 8)
+#define  SOR_TEST_HEAD_MODE_AWAKE               (2 << 8)
+
+#define SOR_PLL_0 0x17
+#define  SOR_PLL_0_ICHPMP_MASK                  (0xf << 24)
+#define  SOR_PLL_0_ICHPMP(x)                    (((x) & 0xf) << 24)
+#define  SOR_PLL_0_VCOCAP_MASK                  (0xf << 8)
+#define  SOR_PLL_0_VCOCAP(x)                    (((x) & 0xf) << 8)
+#define  SOR_PLL_0_VCOCAP_RST                   SOR_PLL_0_VCOCAP(3)
+#define  SOR_PLL_0_PLLREG_MASK                  (0x3 << 6)
+#define  SOR_PLL_0_PLLREG_LEVEL(x)              (((x) & 0x3) << 6)
+#define  SOR_PLL_0_PLLREG_LEVEL_V25             SOR_PLL_0_PLLREG_LEVEL(0)
+#define  SOR_PLL_0_PLLREG_LEVEL_V15             SOR_PLL_0_PLLREG_LEVEL(1)
+#define  SOR_PLL_0_PLLREG_LEVEL_V35             SOR_PLL_0_PLLREG_LEVEL(2)
+#define  SOR_PLL_0_PLLREG_LEVEL_V45             SOR_PLL_0_PLLREG_LEVEL(3)
+#define  SOR_PLL_0_PULLDOWN                     (1 << 5)
+#define  SOR_PLL_0_RESISTOR_EXT                 (1 << 4)
+#define  SOR_PLL_0_VCOPD                        (1 << 2)
+#define  SOR_PLL_0_POWER_OFF                    (1 << 0)
+
+#define SOR_PLL_1 0x18
+/* XXX: read-only bit? */
+#define  SOR_PLL_1_TERM_COMPOUT                 (1 << 15)
+#define  SOR_PLL_1_TMDS_TERM                    (1 << 8)
+
+#define SOR_PLL_2 0x19
+#define  SOR_PLL_2_LVDS_ENABLE                  (1 << 25)
+#define  SOR_PLL_2_SEQ_PLLCAPPD_ENFORCE         (1 << 24)
+#define  SOR_PLL_2_PORT_POWERDOWN               (1 << 23)
+#define  SOR_PLL_2_BANDGAP_POWERDOWN            (1 << 22)
+#define  SOR_PLL_2_POWERDOWN_OVERRIDE           (1 << 18)
+#define  SOR_PLL_2_SEQ_PLLCAPPD                 (1 << 17)
+
+#define SOR_PLL_3 0x1a
+#define  SOR_PLL_3_PLL_VDD_MODE_V1_8 (0 << 13)
+#define  SOR_PLL_3_PLL_VDD_MODE_V3_3 (1 << 13)
+
+#define SOR_CSTM 0x1b
+#define  SOR_CSTM_LVDS                          (1 << 16)
+#define  SOR_CSTM_LINK_ACT_B                    (1 << 15)
+#define  SOR_CSTM_LINK_ACT_A                    (1 << 14)
+#define  SOR_CSTM_UPPER                         (1 << 11)
+
+#define SOR_LVDS 0x1c
+#define SOR_CRC_A 0x1d
+#define SOR_CRC_B 0x1e
+#define SOR_BLANK 0x1f
+#define SOR_SEQ_CTL 0x20
+
+#define SOR_LANE_SEQ_CTL 0x21
+#define  SOR_LANE_SEQ_CTL_TRIGGER               (1 << 31)
+#define  SOR_LANE_SEQ_CTL_SEQUENCE_UP           (0 << 20)
+#define  SOR_LANE_SEQ_CTL_SEQUENCE_DOWN         (1 << 20)
+#define  SOR_LANE_SEQ_CTL_POWER_STATE_UP        (0 << 16)
+#define  SOR_LANE_SEQ_CTL_POWER_STATE_DOWN      (1 << 16)
+
+#define SOR_SEQ_INST(x) (0x22 + (x))
+
+#define SOR_PWM_DIV 0x32
+#define  SOR_PWM_DIV_MASK                       0xffffff
+
+#define SOR_PWM_CTL 0x33
+#define  SOR_PWM_CTL_TRIGGER                    (1 << 31)
+#define  SOR_PWM_CTL_CLK_SEL                    (1 << 30)
+#define  SOR_PWM_CTL_DUTY_CYCLE_MASK            0xffffff
+
+#define SOR_VCRC_A_0 0x34
+#define SOR_VCRC_A_1 0x35
+#define SOR_VCRC_B_0 0x36
+#define SOR_VCRC_B_1 0x37
+#define SOR_CCRC_A_0 0x38
+#define SOR_CCRC_A_1 0x39
+#define SOR_CCRC_B_0 0x3a
+#define SOR_CCRC_B_1 0x3b
+#define SOR_EDATA_A_0 0x3c
+#define SOR_EDATA_A_1 0x3d
+#define SOR_EDATA_B_0 0x3e
+#define SOR_EDATA_B_1 0x3f
+#define SOR_COUNT_A_0 0x40
+#define SOR_COUNT_A_1 0x41
+#define SOR_COUNT_B_0 0x42
+#define SOR_COUNT_B_1 0x43
+#define SOR_DEBUG_A_0 0x44
+#define SOR_DEBUG_A_1 0x45
+#define SOR_DEBUG_B_0 0x46
+#define SOR_DEBUG_B_1 0x47
+#define SOR_TRIG 0x48
+#define SOR_MSCHECK 0x49
+#define SOR_XBAR_CTRL 0x4a
+#define SOR_XBAR_POL 0x4b
+
+#define SOR_DP_LINKCTL_0 0x4c
+#define  SOR_DP_LINKCTL_LANE_COUNT_MASK         (0x1f << 16)
+#define  SOR_DP_LINKCTL_LANE_COUNT(x)           (((1 << (x)) - 1) << 16)
+#define  SOR_DP_LINKCTL_ENHANCED_FRAME          (1 << 14)
+#define  SOR_DP_LINKCTL_TU_SIZE_MASK            (0x7f << 2)
+#define  SOR_DP_LINKCTL_TU_SIZE(x)              (((x) & 0x7f) << 2)
+#define  SOR_DP_LINKCTL_ENABLE                  (1 << 0)
+
+#define SOR_DP_LINKCTL_1 0x4d
+
+#define SOR_LANE_DRIVE_CURRENT_0 0x4e
+#define SOR_LANE_DRIVE_CURRENT_1 0x4f
+#define SOR_LANE4_DRIVE_CURRENT_0 0x50
+#define SOR_LANE4_DRIVE_CURRENT_1 0x51
+#define  SOR_LANE_DRIVE_CURRENT_LANE3(x) (((x) & 0xff) << 24)
+#define  SOR_LANE_DRIVE_CURRENT_LANE2(x) (((x) & 0xff) << 16)
+#define  SOR_LANE_DRIVE_CURRENT_LANE1(x) (((x) & 0xff) << 8)
+#define  SOR_LANE_DRIVE_CURRENT_LANE0(x) (((x) & 0xff) << 0)
+
+#define SOR_LANE_PREEMPHASIS_0 0x52
+#define SOR_LANE_PREEMPHASIS_1 0x53
+#define SOR_LANE4_PREEMPHASIS_0 0x54
+#define SOR_LANE4_PREEMPHASIS_1 0x55
+#define  SOR_LANE_PREEMPHASIS_LANE3(x) (((x) & 0xff) << 24)
+#define  SOR_LANE_PREEMPHASIS_LANE2(x) (((x) & 0xff) << 16)
+#define  SOR_LANE_PREEMPHASIS_LANE1(x) (((x) & 0xff) << 8)
+#define  SOR_LANE_PREEMPHASIS_LANE0(x) (((x) & 0xff) << 0)
+
+#define SOR_LANE_POST_CURSOR_0 0x56
+#define SOR_LANE_POST_CURSOR_1 0x57
+#define  SOR_LANE_POST_CURSOR_LANE3(x) (((x) & 0xff) << 24)
+#define  SOR_LANE_POST_CURSOR_LANE2(x) (((x) & 0xff) << 16)
+#define  SOR_LANE_POST_CURSOR_LANE1(x) (((x) & 0xff) << 8)
+#define  SOR_LANE_POST_CURSOR_LANE0(x) (((x) & 0xff) << 0)
+
+#define SOR_DP_CONFIG_0 0x58
+#define SOR_DP_CONFIG_DISPARITY_NEGATIVE        (1 << 31)
+#define SOR_DP_CONFIG_ACTIVE_SYM_ENABLE         (1 << 26)
+#define SOR_DP_CONFIG_ACTIVE_SYM_POLARITY       (1 << 24)
+#define SOR_DP_CONFIG_ACTIVE_SYM_FRAC_MASK      (0xf << 16)
+#define SOR_DP_CONFIG_ACTIVE_SYM_FRAC(x)        (((x) & 0xf) << 16)
+#define SOR_DP_CONFIG_ACTIVE_SYM_COUNT_MASK     (0x7f << 8)
+#define SOR_DP_CONFIG_ACTIVE_SYM_COUNT(x)       (((x) & 0x7f) << 8)
+#define SOR_DP_CONFIG_WATERMARK_MASK    (0x3f << 0)
+#define SOR_DP_CONFIG_WATERMARK(x)      (((x) & 0x3f) << 0)
+
+#define SOR_DP_CONFIG_1 0x59
+#define SOR_DP_MN_0 0x5a
+#define SOR_DP_MN_1 0x5b
+
+#define SOR_DP_PADCTL_0 0x5c
+#define  SOR_DP_PADCTL_PAD_CAL_PD       (1 << 23)
+#define  SOR_DP_PADCTL_TX_PU_ENABLE     (1 << 22)
+#define  SOR_DP_PADCTL_TX_PU_MASK       (0xff << 8)
+#define  SOR_DP_PADCTL_TX_PU(x)         (((x) & 0xff) << 8)
+#define  SOR_DP_PADCTL_CM_TXD_3         (1 << 7)
+#define  SOR_DP_PADCTL_CM_TXD_2         (1 << 6)
+#define  SOR_DP_PADCTL_CM_TXD_1         (1 << 5)
+#define  SOR_DP_PADCTL_CM_TXD_0         (1 << 4)
+#define  SOR_DP_PADCTL_PD_TXD_3         (1 << 3)
+#define  SOR_DP_PADCTL_PD_TXD_0         (1 << 2)
+#define  SOR_DP_PADCTL_PD_TXD_1         (1 << 1)
+#define  SOR_DP_PADCTL_PD_TXD_2         (1 << 0)
+
+#define SOR_DP_PADCTL_1 0x5d
+
+#define SOR_DP_DEBUG_0 0x5e
+#define SOR_DP_DEBUG_1 0x5f
+
+#define SOR_DP_SPARE_0 0x60
+#define  SOR_DP_SPARE_MACRO_SOR_CLK     (1 << 2)
+#define  SOR_DP_SPARE_PANEL_INTERNAL    (1 << 1)
+#define  SOR_DP_SPARE_SEQ_ENABLE        (1 << 0)
+
+#define SOR_DP_SPARE_1 0x61
+#define SOR_DP_AUDIO_CTRL 0x62
+
+#define SOR_DP_AUDIO_HBLANK_SYMBOLS 0x63
+#define SOR_DP_AUDIO_HBLANK_SYMBOLS_MASK (0x01ffff << 0)
+
+#define SOR_DP_AUDIO_VBLANK_SYMBOLS 0x64
+#define SOR_DP_AUDIO_VBLANK_SYMBOLS_MASK (0x1fffff << 0)
+
+#define SOR_DP_GENERIC_INFOFRAME_HEADER 0x65
+#define SOR_DP_GENERIC_INFOFRAME_SUBPACK_0 0x66
+#define SOR_DP_GENERIC_INFOFRAME_SUBPACK_1 0x67
+#define SOR_DP_GENERIC_INFOFRAME_SUBPACK_2 0x68
+#define SOR_DP_GENERIC_INFOFRAME_SUBPACK_3 0x69
+#define SOR_DP_GENERIC_INFOFRAME_SUBPACK_4 0x6a
+#define SOR_DP_GENERIC_INFOFRAME_SUBPACK_5 0x6b
+#define SOR_DP_GENERIC_INFOFRAME_SUBPACK_6 0x6c
+
+#define SOR_DP_TPG 0x6d
+#define  SOR_DP_TPG_CHANNEL_CODING      (1 << 6)
+#define  SOR_DP_TPG_SCRAMBLER_MASK      (3 << 4)
+#define  SOR_DP_TPG_SCRAMBLER_FIBONACCI (2 << 4)
+#define  SOR_DP_TPG_SCRAMBLER_GALIOS    (1 << 4)
+#define  SOR_DP_TPG_SCRAMBLER_NONE      (0 << 4)
+#define  SOR_DP_TPG_PATTERN_MASK        (0xf << 0)
+#define  SOR_DP_TPG_PATTERN_HBR2        (0x8 << 0)
+#define  SOR_DP_TPG_PATTERN_CSTM        (0x7 << 0)
+#define  SOR_DP_TPG_PATTERN_PRBS7       (0x6 << 0)
+#define  SOR_DP_TPG_PATTERN_SBLERRRATE  (0x5 << 0)
+#define  SOR_DP_TPG_PATTERN_D102        (0x4 << 0)
+#define  SOR_DP_TPG_PATTERN_TRAIN3      (0x3 << 0)
+#define  SOR_DP_TPG_PATTERN_TRAIN2      (0x2 << 0)
+#define  SOR_DP_TPG_PATTERN_TRAIN1      (0x1 << 0)
+#define  SOR_DP_TPG_PATTERN_NONE        (0x0 << 0)
+
+#define SOR_DP_TPG_CONFIG 0x6e
+#define SOR_DP_LQ_CSTM_0 0x6f
+#define SOR_DP_LQ_CSTM_1 0x70
+#define SOR_DP_LQ_CSTM_2 0x71
+
+#endif
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
index d36efc13b16f..d642d4a02134 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
@@ -74,7 +74,7 @@ static void set_scanout(struct drm_crtc *crtc, int n)
                 drm_flip_work_queue(&tilcdc_crtc->unref_work, tilcdc_crtc->scanout[n]);
                 drm_flip_work_commit(&tilcdc_crtc->unref_work, priv->wq);
         }
-        tilcdc_crtc->scanout[n] = crtc->fb;
+        tilcdc_crtc->scanout[n] = crtc->primary->fb;
         drm_framebuffer_reference(tilcdc_crtc->scanout[n]);
         tilcdc_crtc->dirty &= ~stat[n];
         pm_runtime_put_sync(dev->dev);
@@ -84,7 +84,7 @@ static void update_scanout(struct drm_crtc *crtc)
 {
         struct tilcdc_crtc *tilcdc_crtc = to_tilcdc_crtc(crtc);
         struct drm_device *dev = crtc->dev;
-        struct drm_framebuffer *fb = crtc->fb;
+        struct drm_framebuffer *fb = crtc->primary->fb;
         struct drm_gem_cma_object *gem;
         unsigned int depth, bpp;
 
@@ -159,7 +159,7 @@ static int tilcdc_crtc_page_flip(struct drm_crtc *crtc,
                 return -EBUSY;
         }
 
-        crtc->fb = fb;
+        crtc->primary->fb = fb;
         tilcdc_crtc->event = event;
         update_scanout(crtc);
 
@@ -339,7 +339,7 @@ static int tilcdc_crtc_mode_set(struct drm_crtc *crtc,
         if (priv->rev == 2) {
                 unsigned int depth, bpp;
 
-                drm_fb_get_bpp_depth(crtc->fb->pixel_format, &depth, &bpp);
+                drm_fb_get_bpp_depth(crtc->primary->fb->pixel_format, &depth, &bpp);
                 switch (bpp) {
                 case 16:
                         break;
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index 214b7992a3aa..4ab9f7171c4f 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -412,7 +412,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
         int ret;
 
         spin_lock(&glob->lru_lock);
-        ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
+        ret = __ttm_bo_reserve(bo, false, true, false, 0);
 
         spin_lock(&bdev->fence_lock);
         (void) ttm_bo_wait(bo, false, false, true);
@@ -443,7 +443,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo)
                         ttm_bo_add_to_lru(bo);
                 }
 
-                ww_mutex_unlock(&bo->resv->lock);
+                __ttm_bo_unreserve(bo);
         }
 
         kref_get(&bo->list_kref);
@@ -494,7 +494,7 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
                 sync_obj = driver->sync_obj_ref(bo->sync_obj);
                 spin_unlock(&bdev->fence_lock);
 
-                ww_mutex_unlock(&bo->resv->lock);
+                __ttm_bo_unreserve(bo);
                 spin_unlock(&glob->lru_lock);
 
                 ret = driver->sync_obj_wait(sync_obj, false, interruptible);
@@ -514,7 +514,7 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
                         return ret;
 
                 spin_lock(&glob->lru_lock);
-                ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
+                ret = __ttm_bo_reserve(bo, false, true, false, 0);
 
                 /*
                  * We raced, and lost, someone else holds the reservation now,
@@ -532,7 +532,7 @@ static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo,
                 spin_unlock(&bdev->fence_lock);
 
         if (ret || unlikely(list_empty(&bo->ddestroy))) {
-                ww_mutex_unlock(&bo->resv->lock);
+                __ttm_bo_unreserve(bo);
                 spin_unlock(&glob->lru_lock);
                 return ret;
         }
@@ -577,11 +577,11 @@ static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all)
                         kref_get(&nentry->list_kref);
                 }
 
-                ret = ttm_bo_reserve_nolru(entry, false, true, false, 0);
+                ret = __ttm_bo_reserve(entry, false, true, false, 0);
                 if (remove_all && ret) {
                         spin_unlock(&glob->lru_lock);
-                        ret = ttm_bo_reserve_nolru(entry, false, false,
-                                                   false, 0);
+                        ret = __ttm_bo_reserve(entry, false, false,
+                                               false, 0);
                         spin_lock(&glob->lru_lock);
                 }
 
@@ -726,7 +726,7 @@ static int ttm_mem_evict_first(struct ttm_bo_device *bdev,
 
         spin_lock(&glob->lru_lock);
         list_for_each_entry(bo, &man->lru, lru) {
-                ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
+                ret = __ttm_bo_reserve(bo, false, true, false, 0);
                 if (!ret)
                         break;
         }
@@ -1451,6 +1451,7 @@ EXPORT_SYMBOL(ttm_bo_device_release);
 int ttm_bo_device_init(struct ttm_bo_device *bdev,
                        struct ttm_bo_global *glob,
                        struct ttm_bo_driver *driver,
+                       struct address_space *mapping,
                        uint64_t file_page_offset,
                        bool need_dma32)
 {
@@ -1472,7 +1473,7 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev,
                                     0x10000000);
         INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue);
         INIT_LIST_HEAD(&bdev->ddestroy);
-        bdev->dev_mapping = NULL;
+        bdev->dev_mapping = mapping;
         bdev->glob = glob;
         bdev->need_dma32 = need_dma32;
         bdev->val_seq = 0;
@@ -1629,7 +1630,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
 
         spin_lock(&glob->lru_lock);
         list_for_each_entry(bo, &glob->swap_lru, swap) {
-                ret = ttm_bo_reserve_nolru(bo, false, true, false, 0);
+                ret = __ttm_bo_reserve(bo, false, true, false, 0);
                 if (!ret)
                         break;
         }
@@ -1696,7 +1697,7 @@ out:
          * already swapped buffer.
          */
 
-        ww_mutex_unlock(&bo->resv->lock);
+        __ttm_bo_unreserve(bo);
         kref_put(&bo->list_kref, ttm_bo_release_list);
         return ret;
 }
@@ -1730,10 +1731,10 @@ int ttm_bo_wait_unreserved(struct ttm_buffer_object *bo)
                 return -ERESTARTSYS;
         if (!ww_mutex_is_locked(&bo->resv->lock))
                 goto out_unlock;
-        ret = ttm_bo_reserve_nolru(bo, true, false, false, NULL);
+        ret = __ttm_bo_reserve(bo, true, false, false, NULL);
         if (unlikely(ret != 0))
                 goto out_unlock;
-        ww_mutex_unlock(&bo->resv->lock);
+        __ttm_bo_unreserve(bo);
 
 out_unlock:
         mutex_unlock(&bo->wu_mutex);
diff --git a/drivers/gpu/drm/ttm/ttm_bo_manager.c b/drivers/gpu/drm/ttm/ttm_bo_manager.c
index c58eba33bd5f..bd850c9f4bca 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_manager.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_manager.c
@@ -55,6 +55,7 @@ static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man,
         struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv;
         struct drm_mm *mm = &rman->mm;
         struct drm_mm_node *node = NULL;
+        enum drm_mm_allocator_flags aflags = DRM_MM_CREATE_DEFAULT;
         unsigned long lpfn;
         int ret;
 
@@ -66,11 +67,15 @@ static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man,
         if (!node)
                 return -ENOMEM;
 
+        if (bo->mem.placement & TTM_PL_FLAG_TOPDOWN)
+                aflags = DRM_MM_CREATE_TOP;
+
         spin_lock(&rman->lock);
-        ret = drm_mm_insert_node_in_range(mm, node, mem->num_pages,
-                                          mem->page_alignment,
+        ret = drm_mm_insert_node_in_range_generic(mm, node, mem->num_pages,
+                                          mem->page_alignment, 0,
                                           placement->fpfn, lpfn,
-                                          DRM_MM_SEARCH_BEST);
+                                          DRM_MM_SEARCH_BEST,
+                                          aflags);
         spin_unlock(&rman->lock);
 
         if (unlikely(ret)) {
diff --git a/drivers/gpu/drm/ttm/ttm_execbuf_util.c b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
index 479e9418e3d7..e8dac8758528 100644
--- a/drivers/gpu/drm/ttm/ttm_execbuf_util.c
+++ b/drivers/gpu/drm/ttm/ttm_execbuf_util.c
@@ -46,7 +46,7 @@ static void ttm_eu_backoff_reservation_locked(struct list_head *list)
                         ttm_bo_add_to_lru(bo);
                         entry->removed = false;
                 }
-                ww_mutex_unlock(&bo->resv->lock);
+                __ttm_bo_unreserve(bo);
         }
 }
 
@@ -140,8 +140,8 @@ retry:
                 if (entry->reserved)
                         continue;
 
-                ret = ttm_bo_reserve_nolru(bo, true, (ticket == NULL), true,
-                                           ticket);
+                ret = __ttm_bo_reserve(bo, true, (ticket == NULL), true,
+                                       ticket);
 
                 if (ret == -EDEADLK) {
                         /* uh oh, we lost out, drop every reservation and try
@@ -224,7 +224,7 @@ void ttm_eu_fence_buffer_objects(struct ww_acquire_ctx *ticket,
                 entry->old_sync_obj = bo->sync_obj;
                 bo->sync_obj = driver->sync_obj_ref(sync_obj);
                 ttm_bo_add_to_lru(bo);
-                ww_mutex_unlock(&bo->resv->lock);
+                __ttm_bo_unreserve(bo);
                 entry->reserved = false;
         }
         spin_unlock(&bdev->fence_lock);
diff --git a/drivers/gpu/drm/ttm/ttm_object.c b/drivers/gpu/drm/ttm/ttm_object.c
index 53b51c4e671a..d2a053352789 100644
--- a/drivers/gpu/drm/ttm/ttm_object.c
+++ b/drivers/gpu/drm/ttm/ttm_object.c
@@ -270,6 +270,52 @@ ttm_base_object_lookup_for_ref(struct ttm_object_device *tdev, uint32_t key)
 }
 EXPORT_SYMBOL(ttm_base_object_lookup_for_ref);
 
+/**
+ * ttm_ref_object_exists - Check whether a caller has a valid ref object
+ * (has opened) a base object.
+ *
+ * @tfile: Pointer to a struct ttm_object_file identifying the caller.
+ * @base: Pointer to a struct base object.
+ *
+ * Checks wether the caller identified by @tfile has put a valid USAGE
+ * reference object on the base object identified by @base.
+ */
+bool ttm_ref_object_exists(struct ttm_object_file *tfile,
+                           struct ttm_base_object *base)
+{
+        struct drm_open_hash *ht = &tfile->ref_hash[TTM_REF_USAGE];
+        struct drm_hash_item *hash;
+        struct ttm_ref_object *ref;
+
+        rcu_read_lock();
+        if (unlikely(drm_ht_find_item_rcu(ht, base->hash.key, &hash) != 0))
+                goto out_false;
+
+        /*
+         * Verify that the ref object is really pointing to our base object.
+         * Our base object could actually be dead, and the ref object pointing
+         * to another base object with the same handle.
+         */
+        ref = drm_hash_entry(hash, struct ttm_ref_object, hash);
+        if (unlikely(base != ref->obj))
+                goto out_false;
+
+        /*
+         * Verify that the ref->obj pointer was actually valid!
+         */
+        rmb();
+        if (unlikely(atomic_read(&ref->kref.refcount) == 0))
+                goto out_false;
+
+        rcu_read_unlock();
+        return true;
+
+ out_false:
+        rcu_read_unlock();
+        return false;
+}
+EXPORT_SYMBOL(ttm_ref_object_exists);
+
 int ttm_ref_object_add(struct ttm_object_file *tfile,
                        struct ttm_base_object *base,
                        enum ttm_ref_type ref_type, bool *existed)
diff --git a/drivers/gpu/drm/udl/udl_fb.c b/drivers/gpu/drm/udl/udl_fb.c
index dbadd49e4c4a..377176372da8 100644
--- a/drivers/gpu/drm/udl/udl_fb.c
+++ b/drivers/gpu/drm/udl/udl_fb.c
@@ -421,7 +421,7 @@ static int udl_user_framebuffer_dirty(struct drm_framebuffer *fb,
                                   clips[i].x2 - clips[i].x1,
                                   clips[i].y2 - clips[i].y1);
                 if (ret)
-                        goto unlock;
+                        break;
         }
 
         if (ufb->obj->base.import_attach) {
diff --git a/drivers/gpu/drm/udl/udl_gem.c b/drivers/gpu/drm/udl/udl_gem.c
index 0394811251bd..c041cd73f399 100644
--- a/drivers/gpu/drm/udl/udl_gem.c
+++ b/drivers/gpu/drm/udl/udl_gem.c
@@ -60,7 +60,7 @@ int udl_dumb_create(struct drm_file *file,
                     struct drm_device *dev,
                     struct drm_mode_create_dumb *args)
 {
-        args->pitch = args->width * ((args->bpp + 1) / 8);
+        args->pitch = args->width * DIV_ROUND_UP(args->bpp, 8);
         args->size = args->pitch * args->height;
         return udl_gem_create(file, dev,
                               args->size, &args->handle);
diff --git a/drivers/gpu/drm/udl/udl_modeset.c b/drivers/gpu/drm/udl/udl_modeset.c
index 2ae1eb7d1635..cddc4fcf35cf 100644
--- a/drivers/gpu/drm/udl/udl_modeset.c
+++ b/drivers/gpu/drm/udl/udl_modeset.c
@@ -310,7 +310,7 @@ static int udl_crtc_mode_set(struct drm_crtc *crtc,
 
 {
         struct drm_device *dev = crtc->dev;
-        struct udl_framebuffer *ufb = to_udl_fb(crtc->fb);
+        struct udl_framebuffer *ufb = to_udl_fb(crtc->primary->fb);
         struct udl_device *udl = dev->dev_private;
         char *buf;
         char *wrptr;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_context.c b/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
index 1e80152674b5..8bb26dcd9eae 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_context.c
@@ -117,10 +117,10 @@ static void vmw_hw_context_destroy(struct vmw_resource *res)
                 (void) vmw_context_binding_state_kill
                         (&container_of(res, struct vmw_user_context, res)->cbs);
                 (void) vmw_gb_context_destroy(res);
+                mutex_unlock(&dev_priv->binding_mutex);
                 if (dev_priv->pinned_bo != NULL &&
                     !dev_priv->query_cid_valid)
                         __vmw_execbuf_release_pinned_bo(dev_priv, NULL);
-                mutex_unlock(&dev_priv->binding_mutex);
                 mutex_unlock(&dev_priv->cmdbuf_mutex);
                 return;
         }
@@ -462,7 +462,6 @@ int vmw_context_define_ioctl(struct drm_device *dev, void *data,
         struct vmw_resource *tmp;
         struct drm_vmw_context_arg *arg = (struct drm_vmw_context_arg *)data;
         struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
-        struct vmw_master *vmaster = vmw_master(file_priv->master);
         int ret;
 
 
@@ -474,7 +473,7 @@ int vmw_context_define_ioctl(struct drm_device *dev, void *data,
         if (unlikely(vmw_user_context_size == 0))
                 vmw_user_context_size = ttm_round_pot(sizeof(*ctx)) + 128;
 
-        ret = ttm_read_lock(&vmaster->lock, true);
+        ret = ttm_read_lock(&dev_priv->reservation_sem, true);
         if (unlikely(ret != 0))
                 return ret;
 
@@ -521,7 +520,7 @@ int vmw_context_define_ioctl(struct drm_device *dev, void *data,
 out_err:
         vmw_resource_unreference(&res);
 out_unlock:
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
         return ret;
 
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c
index a75840211b3c..70ddce8358b0 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_dmabuf.c
@@ -52,11 +52,10 @@ int vmw_dmabuf_to_placement(struct vmw_private *dev_priv,
                             struct ttm_placement *placement,
                             bool interruptible)
 {
-        struct vmw_master *vmaster = dev_priv->active_master;
         struct ttm_buffer_object *bo = &buf->base;
         int ret;
 
-        ret = ttm_write_lock(&vmaster->lock, interruptible);
+        ret = ttm_write_lock(&dev_priv->reservation_sem, interruptible);
         if (unlikely(ret != 0))
                 return ret;
 
@@ -71,7 +70,7 @@ int vmw_dmabuf_to_placement(struct vmw_private *dev_priv,
         ttm_bo_unreserve(bo);
 
 err:
-        ttm_write_unlock(&vmaster->lock);
+        ttm_write_unlock(&dev_priv->reservation_sem);
         return ret;
 }
 
@@ -95,12 +94,11 @@ int vmw_dmabuf_to_vram_or_gmr(struct vmw_private *dev_priv,
                               struct vmw_dma_buffer *buf,
                               bool pin, bool interruptible)
 {
-        struct vmw_master *vmaster = dev_priv->active_master;
         struct ttm_buffer_object *bo = &buf->base;
         struct ttm_placement *placement;
         int ret;
 
-        ret = ttm_write_lock(&vmaster->lock, interruptible);
+        ret = ttm_write_lock(&dev_priv->reservation_sem, interruptible);
         if (unlikely(ret != 0))
                 return ret;
 
@@ -143,7 +141,7 @@ int vmw_dmabuf_to_vram_or_gmr(struct vmw_private *dev_priv,
 err_unreserve:
         ttm_bo_unreserve(bo);
 err:
-        ttm_write_unlock(&vmaster->lock);
+        ttm_write_unlock(&dev_priv->reservation_sem);
         return ret;
 }
 
@@ -198,7 +196,6 @@ int vmw_dmabuf_to_start_of_vram(struct vmw_private *dev_priv,
                                 struct vmw_dma_buffer *buf,
                                 bool pin, bool interruptible)
 {
-        struct vmw_master *vmaster = dev_priv->active_master;
         struct ttm_buffer_object *bo = &buf->base;
         struct ttm_placement placement;
         int ret = 0;
@@ -209,7 +206,7 @@ int vmw_dmabuf_to_start_of_vram(struct vmw_private *dev_priv,
                 placement = vmw_vram_placement;
         placement.lpfn = bo->num_pages;
 
-        ret = ttm_write_lock(&vmaster->lock, interruptible);
+        ret = ttm_write_lock(&dev_priv->reservation_sem, interruptible);
         if (unlikely(ret != 0))
                 return ret;
 
@@ -232,7 +229,7 @@ int vmw_dmabuf_to_start_of_vram(struct vmw_private *dev_priv,
 
         ttm_bo_unreserve(bo);
 err_unlock:
-        ttm_write_unlock(&vmaster->lock);
+        ttm_write_unlock(&dev_priv->reservation_sem);
 
         return ret;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index 0083cbf99edf..4a223bbea3b3 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -142,11 +142,11 @@
 
 static const struct drm_ioctl_desc vmw_ioctls[] = {
         VMW_IOCTL_DEF(VMW_GET_PARAM, vmw_getparam_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_ALLOC_DMABUF, vmw_dmabuf_alloc_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_UNREF_DMABUF, vmw_dmabuf_unref_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_CURSOR_BYPASS,
                       vmw_kms_cursor_bypass_ioctl,
                       DRM_MASTER | DRM_CONTROL_ALLOW | DRM_UNLOCKED),
@@ -159,29 +159,28 @@ static const struct drm_ioctl_desc vmw_ioctls[] = {
                       DRM_MASTER | DRM_CONTROL_ALLOW | DRM_UNLOCKED),
 
         VMW_IOCTL_DEF(VMW_CREATE_CONTEXT, vmw_context_define_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_UNREF_CONTEXT, vmw_context_destroy_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_CREATE_SURFACE, vmw_surface_define_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_UNREF_SURFACE, vmw_surface_destroy_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_REF_SURFACE, vmw_surface_reference_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_EXECBUF, vmw_execbuf_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_FENCE_WAIT, vmw_fence_obj_wait_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_FENCE_SIGNALED,
                       vmw_fence_obj_signaled_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_FENCE_UNREF, vmw_fence_obj_unref_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
-        VMW_IOCTL_DEF(VMW_FENCE_EVENT,
-                      vmw_fence_event_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_UNLOCKED | DRM_RENDER_ALLOW),
+        VMW_IOCTL_DEF(VMW_FENCE_EVENT, vmw_fence_event_ioctl,
+                      DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_GET_3D_CAP, vmw_get_cap_3d_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
 
         /* these allow direct access to the framebuffers mark as master only */
         VMW_IOCTL_DEF(VMW_PRESENT, vmw_present_ioctl,
@@ -194,19 +193,19 @@ static const struct drm_ioctl_desc vmw_ioctls[] = {
                       DRM_MASTER | DRM_UNLOCKED),
         VMW_IOCTL_DEF(VMW_CREATE_SHADER,
                       vmw_shader_define_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_UNREF_SHADER,
                       vmw_shader_destroy_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_GB_SURFACE_CREATE,
                       vmw_gb_surface_define_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_GB_SURFACE_REF,
                       vmw_gb_surface_reference_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_AUTH | DRM_UNLOCKED | DRM_RENDER_ALLOW),
         VMW_IOCTL_DEF(VMW_SYNCCPU,
                       vmw_user_dmabuf_synccpu_ioctl,
-                      DRM_AUTH | DRM_UNLOCKED),
+                      DRM_UNLOCKED | DRM_RENDER_ALLOW),
 };
 
 static struct pci_device_id vmw_pci_id_list[] = {
@@ -606,6 +605,7 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
         mutex_init(&dev_priv->release_mutex);
         mutex_init(&dev_priv->binding_mutex);
         rwlock_init(&dev_priv->resource_lock);
+        ttm_lock_init(&dev_priv->reservation_sem);
 
         for (i = vmw_res_context; i < vmw_res_max; ++i) {
                 idr_init(&dev_priv->res_idr[i]);
@@ -722,7 +722,9 @@ static int vmw_driver_load(struct drm_device *dev, unsigned long chipset)
 
         ret = ttm_bo_device_init(&dev_priv->bdev,
                                  dev_priv->bo_global_ref.ref.object,
-                                 &vmw_bo_driver, VMWGFX_FILE_PAGE_OFFSET,
+                                 &vmw_bo_driver,
+                                 dev->anon_inode->i_mapping,
+                                 VMWGFX_FILE_PAGE_OFFSET,
                                  false);
         if (unlikely(ret != 0)) {
                 DRM_ERROR("Failed initializing TTM buffer object driver.\n");
@@ -969,7 +971,6 @@ static int vmw_driver_open(struct drm_device *dev, struct drm_file *file_priv)
                 goto out_no_shman;
 
         file_priv->driver_priv = vmw_fp;
-        dev_priv->bdev.dev_mapping = dev->dev_mapping;
 
         return 0;
 
@@ -980,12 +981,70 @@ out_no_tfile:
         return ret;
 }
 
-static long vmw_unlocked_ioctl(struct file *filp, unsigned int cmd,
-                               unsigned long arg)
+static struct vmw_master *vmw_master_check(struct drm_device *dev,
+                                           struct drm_file *file_priv,
+                                           unsigned int flags)
+{
+        int ret;
+        struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
+        struct vmw_master *vmaster;
+
+        if (file_priv->minor->type != DRM_MINOR_LEGACY ||
+            !(flags & DRM_AUTH))
+                return NULL;
+
+        ret = mutex_lock_interruptible(&dev->master_mutex);
+        if (unlikely(ret != 0))
+                return ERR_PTR(-ERESTARTSYS);
+
+        if (file_priv->is_master) {
+                mutex_unlock(&dev->master_mutex);
+                return NULL;
+        }
+
+        /*
+         * Check if we were previously master, but now dropped.
+         */
+        if (vmw_fp->locked_master) {
+                mutex_unlock(&dev->master_mutex);
+                DRM_ERROR("Dropped master trying to access ioctl that "
+                          "requires authentication.\n");
+                return ERR_PTR(-EACCES);
+        }
+        mutex_unlock(&dev->master_mutex);
+
+        /*
+         * Taking the drm_global_mutex after the TTM lock might deadlock
+         */
+        if (!(flags & DRM_UNLOCKED)) {
+                DRM_ERROR("Refusing locked ioctl access.\n");
+                return ERR_PTR(-EDEADLK);
+        }
+
+        /*
+         * Take the TTM lock. Possibly sleep waiting for the authenticating
+         * master to become master again, or for a SIGTERM if the
+         * authenticating master exits.
+         */
+        vmaster = vmw_master(file_priv->master);
+        ret = ttm_read_lock(&vmaster->lock, true);
+        if (unlikely(ret != 0))
+                vmaster = ERR_PTR(ret);
+
+        return vmaster;
+}
+
+static long vmw_generic_ioctl(struct file *filp, unsigned int cmd,
+                              unsigned long arg,
+                              long (*ioctl_func)(struct file *, unsigned int,
+                                                 unsigned long))
 {
         struct drm_file *file_priv = filp->private_data;
         struct drm_device *dev = file_priv->minor->dev;
         unsigned int nr = DRM_IOCTL_NR(cmd);
+        struct vmw_master *vmaster;
+        unsigned int flags;
+        long ret;
 
         /*
          * Do extra checking on driver private ioctls.
@@ -994,18 +1053,44 @@ static long vmw_unlocked_ioctl(struct file *filp, unsigned int cmd,
         if ((nr >= DRM_COMMAND_BASE) && (nr < DRM_COMMAND_END)
             && (nr < DRM_COMMAND_BASE + dev->driver->num_ioctls)) {
                 const struct drm_ioctl_desc *ioctl =
-                    &vmw_ioctls[nr - DRM_COMMAND_BASE];
+                        &vmw_ioctls[nr - DRM_COMMAND_BASE];
 
                 if (unlikely(ioctl->cmd_drv != cmd)) {
                         DRM_ERROR("Invalid command format, ioctl %d\n",
                                   nr - DRM_COMMAND_BASE);
                         return -EINVAL;
                 }
+                flags = ioctl->flags;
+        } else if (!drm_ioctl_flags(nr, &flags))
+                return -EINVAL;
+
+        vmaster = vmw_master_check(dev, file_priv, flags);
+        if (unlikely(IS_ERR(vmaster))) {
+                DRM_INFO("IOCTL ERROR %d\n", nr);
+                return PTR_ERR(vmaster);
         }
 
-        return drm_ioctl(filp, cmd, arg);
+        ret = ioctl_func(filp, cmd, arg);
+        if (vmaster)
+                ttm_read_unlock(&vmaster->lock);
+
+        return ret;
+}
+
+static long vmw_unlocked_ioctl(struct file *filp, unsigned int cmd,
+                               unsigned long arg)
+{
+        return vmw_generic_ioctl(filp, cmd, arg, &drm_ioctl);
 }
 
+#ifdef CONFIG_COMPAT
+static long vmw_compat_ioctl(struct file *filp, unsigned int cmd,
+                             unsigned long arg)
+{
+        return vmw_generic_ioctl(filp, cmd, arg, &drm_compat_ioctl);
+}
+#endif
+
 static void vmw_lastclose(struct drm_device *dev)
 {
         struct drm_crtc *crtc;
@@ -1174,12 +1259,11 @@ static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val,
 {
         struct vmw_private *dev_priv =
                 container_of(nb, struct vmw_private, pm_nb);
-        struct vmw_master *vmaster = dev_priv->active_master;
 
         switch (val) {
         case PM_HIBERNATION_PREPARE:
         case PM_SUSPEND_PREPARE:
-                ttm_suspend_lock(&vmaster->lock);
+                ttm_suspend_lock(&dev_priv->reservation_sem);
 
                 /**
                  * This empties VRAM and unbinds all GMR bindings.
@@ -1193,7 +1277,7 @@ static int vmwgfx_pm_notifier(struct notifier_block *nb, unsigned long val,
         case PM_POST_HIBERNATION:
         case PM_POST_SUSPEND:
         case PM_POST_RESTORE:
-                ttm_suspend_unlock(&vmaster->lock);
+                ttm_suspend_unlock(&dev_priv->reservation_sem);
 
                 break;
         case PM_RESTORE_PREPARE:
@@ -1314,14 +1398,14 @@ static const struct file_operations vmwgfx_driver_fops = {
         .poll = vmw_fops_poll,
         .read = vmw_fops_read,
 #if defined(CONFIG_COMPAT)
-        .compat_ioctl = drm_compat_ioctl,
+        .compat_ioctl = vmw_compat_ioctl,
 #endif
         .llseek = noop_llseek,
 };
 
 static struct drm_driver driver = {
         .driver_features = DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED |
-        DRIVER_MODESET | DRIVER_PRIME,
+        DRIVER_MODESET | DRIVER_PRIME | DRIVER_RENDER,
         .load = vmw_driver_load,
         .unload = vmw_driver_unload,
         .lastclose = vmw_lastclose,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index 07831554dad7..6b252a887ae2 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -40,9 +40,9 @@
 #include <drm/ttm/ttm_module.h>
 #include "vmwgfx_fence.h"
 
-#define VMWGFX_DRIVER_DATE "20140228"
+#define VMWGFX_DRIVER_DATE "20140325"
 #define VMWGFX_DRIVER_MAJOR 2
-#define VMWGFX_DRIVER_MINOR 5
+#define VMWGFX_DRIVER_MINOR 6
 #define VMWGFX_DRIVER_PATCHLEVEL 0
 #define VMWGFX_FILE_PAGE_OFFSET 0x00100000
 #define VMWGFX_FIFO_STATIC_SIZE (1024*1024)
@@ -487,6 +487,11 @@ struct vmw_private {
         uint32_t num_3d_resources;
 
         /*
+         * Replace this with an rwsem as soon as we have down_xx_interruptible()
+         */
+        struct ttm_lock reservation_sem;
+
+        /*
          * Query processing. These members
          * are protected by the cmdbuf mutex.
          */
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
index efb575a7996c..931490b9cfed 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
@@ -2712,7 +2712,6 @@ int vmw_execbuf_ioctl(struct drm_device *dev, void *data,
 {
         struct vmw_private *dev_priv = vmw_priv(dev);
         struct drm_vmw_execbuf_arg *arg = (struct drm_vmw_execbuf_arg *)data;
-        struct vmw_master *vmaster = vmw_master(file_priv->master);
         int ret;
 
         /*
@@ -2729,7 +2728,7 @@ int vmw_execbuf_ioctl(struct drm_device *dev, void *data,
                 return -EINVAL;
         }
 
-        ret = ttm_read_lock(&vmaster->lock, true);
+        ret = ttm_read_lock(&dev_priv->reservation_sem, true);
         if (unlikely(ret != 0))
                 return ret;
 
@@ -2745,6 +2744,6 @@ int vmw_execbuf_ioctl(struct drm_device *dev, void *data,
         vmw_kms_cursor_post_execbuf(dev_priv);
 
 out_unlock:
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
         return ret;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
index ed5ce2a41bbf..a89ad938eacf 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fb.c
@@ -147,7 +147,7 @@ static int vmw_fb_check_var(struct fb_var_screeninfo *var,
         }
 
         if (!vmw_kms_validate_mode_vram(vmw_priv,
-                                        info->fix.line_length,
+                                        var->xres * var->bits_per_pixel/8,
                                         var->yoffset + var->yres)) {
                 DRM_ERROR("Requested geom can not fit in framebuffer\n");
                 return -EINVAL;
@@ -162,6 +162,8 @@ static int vmw_fb_set_par(struct fb_info *info)
         struct vmw_private *vmw_priv = par->vmw_priv;
         int ret;
 
+        info->fix.line_length = info->var.xres * info->var.bits_per_pixel/8;
+
         ret = vmw_kms_write_svga(vmw_priv, info->var.xres, info->var.yres,
                                  info->fix.line_length,
                                  par->bpp, par->depth);
@@ -177,6 +179,7 @@ static int vmw_fb_set_par(struct fb_info *info)
                 vmw_write(vmw_priv, SVGA_REG_DISPLAY_POSITION_Y, info->var.yoffset);
                 vmw_write(vmw_priv, SVGA_REG_DISPLAY_WIDTH, info->var.xres);
                 vmw_write(vmw_priv, SVGA_REG_DISPLAY_HEIGHT, info->var.yres);
+                vmw_write(vmw_priv, SVGA_REG_BYTES_PER_LINE, info->fix.line_length);
                 vmw_write(vmw_priv, SVGA_REG_DISPLAY_ID, SVGA_ID_INVALID);
         }
 
@@ -377,14 +380,13 @@ static int vmw_fb_create_bo(struct vmw_private *vmw_priv,
 
         ne_placement.lpfn = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
 
-        /* interuptable? */
-        ret = ttm_write_lock(&vmw_priv->fbdev_master.lock, false);
-        if (unlikely(ret != 0))
-                return ret;
+        (void) ttm_write_lock(&vmw_priv->reservation_sem, false);
 
         vmw_bo = kmalloc(sizeof(*vmw_bo), GFP_KERNEL);
-        if (!vmw_bo)
+        if (!vmw_bo) {
+                ret = -ENOMEM;
                 goto err_unlock;
+        }
 
         ret = vmw_dmabuf_init(vmw_priv, vmw_bo, size,
                               &ne_placement,
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
index 47b70949bf3a..37881ecf5d7a 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ioctl.c
@@ -226,7 +226,6 @@ int vmw_present_ioctl(struct drm_device *dev, void *data,
         struct drm_vmw_present_arg *arg =
                 (struct drm_vmw_present_arg *)data;
         struct vmw_surface *surface;
-        struct vmw_master *vmaster = vmw_master(file_priv->master);
         struct drm_vmw_rect __user *clips_ptr;
         struct drm_vmw_rect *clips = NULL;
         struct drm_framebuffer *fb;
@@ -271,7 +270,7 @@ int vmw_present_ioctl(struct drm_device *dev, void *data,
         }
         vfb = vmw_framebuffer_to_vfb(fb);
 
-        ret = ttm_read_lock(&vmaster->lock, true);
+        ret = ttm_read_lock(&dev_priv->reservation_sem, true);
         if (unlikely(ret != 0))
                 goto out_no_ttm_lock;
 
@@ -291,7 +290,7 @@ int vmw_present_ioctl(struct drm_device *dev, void *data,
         vmw_surface_unreference(&surface);
 
 out_no_surface:
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
 out_no_ttm_lock:
         drm_framebuffer_unreference(fb);
 out_no_fb:
@@ -311,7 +310,6 @@ int vmw_present_readback_ioctl(struct drm_device *dev, void *data,
         struct drm_vmw_fence_rep __user *user_fence_rep =
                 (struct drm_vmw_fence_rep __user *)
                 (unsigned long)arg->fence_rep;
-        struct vmw_master *vmaster = vmw_master(file_priv->master);
         struct drm_vmw_rect __user *clips_ptr;
         struct drm_vmw_rect *clips = NULL;
         struct drm_framebuffer *fb;
@@ -361,7 +359,7 @@ int vmw_present_readback_ioctl(struct drm_device *dev, void *data,
                 goto out_no_ttm_lock;
         }
 
-        ret = ttm_read_lock(&vmaster->lock, true);
+        ret = ttm_read_lock(&dev_priv->reservation_sem, true);
         if (unlikely(ret != 0))
                 goto out_no_ttm_lock;
 
@@ -369,7 +367,7 @@ int vmw_present_readback_ioctl(struct drm_device *dev, void *data,
                                vfb, user_fence_rep,
                                clips, num_clips);
 
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
 out_no_ttm_lock:
         drm_framebuffer_unreference(fb);
 out_no_fb:
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index 8a650413dea5..a2dde5ad8138 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -468,7 +468,7 @@ static int do_surface_dirty_sou(struct vmw_private *dev_priv,
         num_units = 0;
         list_for_each_entry(crtc, &dev_priv->dev->mode_config.crtc_list,
                             head) {
-                if (crtc->fb != &framebuffer->base)
+                if (crtc->primary->fb != &framebuffer->base)
                         continue;
                 units[num_units++] = vmw_crtc_to_du(crtc);
         }
@@ -596,7 +596,6 @@ static int vmw_framebuffer_surface_dirty(struct drm_framebuffer *framebuffer,
                                   unsigned num_clips)
 {
         struct vmw_private *dev_priv = vmw_priv(framebuffer->dev);
-        struct vmw_master *vmaster = vmw_master(file_priv->master);
         struct vmw_framebuffer_surface *vfbs =
                 vmw_framebuffer_to_vfbs(framebuffer);
         struct drm_clip_rect norect;
@@ -611,7 +610,7 @@ static int vmw_framebuffer_surface_dirty(struct drm_framebuffer *framebuffer,
 
         drm_modeset_lock_all(dev_priv->dev);
 
-        ret = ttm_read_lock(&vmaster->lock, true);
+        ret = ttm_read_lock(&dev_priv->reservation_sem, true);
         if (unlikely(ret != 0)) {
                 drm_modeset_unlock_all(dev_priv->dev);
                 return ret;
@@ -632,7 +631,7 @@ static int vmw_framebuffer_surface_dirty(struct drm_framebuffer *framebuffer,
                                    flags, color,
                                    clips, num_clips, inc, NULL);
 
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
 
         drm_modeset_unlock_all(dev_priv->dev);
 
@@ -883,7 +882,7 @@ static int do_dmabuf_dirty_sou(struct drm_file *file_priv,
 
         num_units = 0;
         list_for_each_entry(crtc, &dev_priv->dev->mode_config.crtc_list, head) {
-                if (crtc->fb != &framebuffer->base)
+                if (crtc->primary->fb != &framebuffer->base)
                         continue;
                 units[num_units++] = vmw_crtc_to_du(crtc);
         }
@@ -954,7 +953,6 @@ static int vmw_framebuffer_dmabuf_dirty(struct drm_framebuffer *framebuffer,
                                  unsigned num_clips)
 {
         struct vmw_private *dev_priv = vmw_priv(framebuffer->dev);
-        struct vmw_master *vmaster = vmw_master(file_priv->master);
         struct vmw_framebuffer_dmabuf *vfbd =
                 vmw_framebuffer_to_vfbd(framebuffer);
         struct drm_clip_rect norect;
@@ -962,7 +960,7 @@ static int vmw_framebuffer_dmabuf_dirty(struct drm_framebuffer *framebuffer,
 
         drm_modeset_lock_all(dev_priv->dev);
 
-        ret = ttm_read_lock(&vmaster->lock, true);
+        ret = ttm_read_lock(&dev_priv->reservation_sem, true);
         if (unlikely(ret != 0)) {
                 drm_modeset_unlock_all(dev_priv->dev);
                 return ret;
@@ -989,7 +987,7 @@ static int vmw_framebuffer_dmabuf_dirty(struct drm_framebuffer *framebuffer,
                                           clips, num_clips, increment, NULL);
         }
 
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
 
         drm_modeset_unlock_all(dev_priv->dev);
 
@@ -1245,7 +1243,7 @@ int vmw_kms_present(struct vmw_private *dev_priv,
 
         num_units = 0;
         list_for_each_entry(crtc, &dev_priv->dev->mode_config.crtc_list, head) {
-                if (crtc->fb != &vfb->base)
+                if (crtc->primary->fb != &vfb->base)
                         continue;
                 units[num_units++] = vmw_crtc_to_du(crtc);
         }
@@ -1382,7 +1380,7 @@ int vmw_kms_readback(struct vmw_private *dev_priv,
 
         num_units = 0;
         list_for_each_entry(crtc, &dev_priv->dev->mode_config.crtc_list, head) {
-                if (crtc->fb != &vfb->base)
+                if (crtc->primary->fb != &vfb->base)
                         continue;
                 units[num_units++] = vmw_crtc_to_du(crtc);
         }
@@ -1725,7 +1723,7 @@ int vmw_du_page_flip(struct drm_crtc *crtc,
                      uint32_t page_flip_flags)
 {
         struct vmw_private *dev_priv = vmw_priv(crtc->dev);
-        struct drm_framebuffer *old_fb = crtc->fb;
+        struct drm_framebuffer *old_fb = crtc->primary->fb;
         struct vmw_framebuffer *vfb = vmw_framebuffer_to_vfb(fb);
         struct drm_file *file_priv ;
         struct vmw_fence_obj *fence = NULL;
@@ -1743,7 +1741,7 @@ int vmw_du_page_flip(struct drm_crtc *crtc,
         if (!vmw_kms_screen_object_flippable(dev_priv, crtc))
                 return -EINVAL;
 
-        crtc->fb = fb;
+        crtc->primary->fb = fb;
 
         /* do a full screen dirty update */
         clips.x1 = clips.y1 = 0;
@@ -1783,7 +1781,7 @@ int vmw_du_page_flip(struct drm_crtc *crtc,
         return ret;
 
 out_no_fence:
-        crtc->fb = old_fb;
+        crtc->primary->fb = old_fb;
         return ret;
 }
 
@@ -2022,7 +2020,6 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
         struct vmw_private *dev_priv = vmw_priv(dev);
         struct drm_vmw_update_layout_arg *arg =
                 (struct drm_vmw_update_layout_arg *)data;
-        struct vmw_master *vmaster = vmw_master(file_priv->master);
         void __user *user_rects;
         struct drm_vmw_rect *rects;
         unsigned rects_size;
@@ -2030,7 +2027,7 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
         int i;
         struct drm_mode_config *mode_config = &dev->mode_config;
 
-        ret = ttm_read_lock(&vmaster->lock, true);
+        ret = ttm_read_lock(&dev_priv->reservation_sem, true);
         if (unlikely(ret != 0))
                 return ret;
 
@@ -2072,6 +2069,6 @@ int vmw_kms_update_layout_ioctl(struct drm_device *dev, void *data,
 out_free:
         kfree(rects);
 out_unlock:
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
         return ret;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
index a055a26819c2..b2b9bd23aeee 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_ldu.c
@@ -93,7 +93,7 @@ static int vmw_ldu_commit_list(struct vmw_private *dev_priv)
 
                 if (crtc == NULL)
                         return 0;
-                fb = entry->base.crtc.fb;
+                fb = entry->base.crtc.primary->fb;
 
                 return vmw_kms_write_svga(dev_priv, w, h, fb->pitches[0],
                                           fb->bits_per_pixel, fb->depth);
@@ -101,7 +101,7 @@ static int vmw_ldu_commit_list(struct vmw_private *dev_priv)
 
         if (!list_empty(&lds->active)) {
                 entry = list_entry(lds->active.next, typeof(*entry), active);
-                fb = entry->base.crtc.fb;
+                fb = entry->base.crtc.primary->fb;
 
                 vmw_kms_write_svga(dev_priv, fb->width, fb->height, fb->pitches[0],
                                    fb->bits_per_pixel, fb->depth);
@@ -259,7 +259,7 @@ static int vmw_ldu_crtc_set_config(struct drm_mode_set *set)
 
                 connector->encoder = NULL;
                 encoder->crtc = NULL;
-                crtc->fb = NULL;
+                crtc->primary->fb = NULL;
                 crtc->enabled = false;
 
                 vmw_ldu_del_active(dev_priv, ldu);
@@ -280,7 +280,7 @@ static int vmw_ldu_crtc_set_config(struct drm_mode_set *set)
 
         vmw_fb_off(dev_priv);
 
-        crtc->fb = fb;
+        crtc->primary->fb = fb;
         encoder->crtc = crtc;
         connector->encoder = encoder;
         crtc->x = set->x;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
index 9757b57f8388..01d68f0a69dc 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_resource.c
@@ -538,8 +538,13 @@ int vmw_user_dmabuf_verify_access(struct ttm_buffer_object *bo,
                 return -EPERM;
 
         vmw_user_bo = vmw_user_dma_buffer(bo);
-        return (vmw_user_bo->prime.base.tfile == tfile ||
-                vmw_user_bo->prime.base.shareable) ? 0 : -EPERM;
+
+        /* Check that the caller has opened the object. */
+        if (likely(ttm_ref_object_exists(tfile, &vmw_user_bo->prime.base)))
+                return 0;
+
+        DRM_ERROR("Could not grant buffer access.\n");
+        return -EPERM;
 }
 
 /**
@@ -676,10 +681,9 @@ int vmw_dmabuf_alloc_ioctl(struct drm_device *dev, void *data,
         struct drm_vmw_dmabuf_rep *rep = &arg->rep;
         struct vmw_dma_buffer *dma_buf;
         uint32_t handle;
-        struct vmw_master *vmaster = vmw_master(file_priv->master);
         int ret;
 
-        ret = ttm_read_lock(&vmaster->lock, true);
+        ret = ttm_read_lock(&dev_priv->reservation_sem, true);
         if (unlikely(ret != 0))
                 return ret;
 
@@ -696,7 +700,7 @@ int vmw_dmabuf_alloc_ioctl(struct drm_device *dev, void *data,
         vmw_dmabuf_unreference(&dma_buf);
 
 out_no_dmabuf:
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
 
         return ret;
 }
@@ -873,7 +877,6 @@ int vmw_stream_claim_ioctl(struct drm_device *dev, void *data,
         struct vmw_resource *tmp;
         struct drm_vmw_stream_arg *arg = (struct drm_vmw_stream_arg *)data;
         struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
-        struct vmw_master *vmaster = vmw_master(file_priv->master);
         int ret;
 
         /*
@@ -884,7 +887,7 @@ int vmw_stream_claim_ioctl(struct drm_device *dev, void *data,
         if (unlikely(vmw_user_stream_size == 0))
                 vmw_user_stream_size = ttm_round_pot(sizeof(*stream)) + 128;
 
-        ret = ttm_read_lock(&vmaster->lock, true);
+        ret = ttm_read_lock(&dev_priv->reservation_sem, true);
         if (unlikely(ret != 0))
                 return ret;
 
@@ -932,7 +935,7 @@ int vmw_stream_claim_ioctl(struct drm_device *dev, void *data,
 out_err:
         vmw_resource_unreference(&res);
 out_unlock:
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
         return ret;
 }
 
@@ -985,14 +988,13 @@ int vmw_dumb_create(struct drm_file *file_priv,
                     struct drm_mode_create_dumb *args)
 {
         struct vmw_private *dev_priv = vmw_priv(dev);
-        struct vmw_master *vmaster = vmw_master(file_priv->master);
         struct vmw_dma_buffer *dma_buf;
         int ret;
 
         args->pitch = args->width * ((args->bpp + 7) / 8);
         args->size = args->pitch * args->height;
 
-        ret = ttm_read_lock(&vmaster->lock, true);
+        ret = ttm_read_lock(&dev_priv->reservation_sem, true);
         if (unlikely(ret != 0))
                 return ret;
 
@@ -1004,7 +1006,7 @@ int vmw_dumb_create(struct drm_file *file_priv,
 
         vmw_dmabuf_unreference(&dma_buf);
 out_no_dmabuf:
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
         return ret;
 }
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
index 22406c8651ea..a95d3a0cabe4 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_scrn.c
@@ -307,7 +307,7 @@ static int vmw_sou_crtc_set_config(struct drm_mode_set *set)
 
                 connector->encoder = NULL;
                 encoder->crtc = NULL;
-                crtc->fb = NULL;
+                crtc->primary->fb = NULL;
                 crtc->x = 0;
                 crtc->y = 0;
                 crtc->enabled = false;
@@ -368,7 +368,7 @@ static int vmw_sou_crtc_set_config(struct drm_mode_set *set)
 
                 connector->encoder = NULL;
                 encoder->crtc = NULL;
-                crtc->fb = NULL;
+                crtc->primary->fb = NULL;
                 crtc->x = 0;
                 crtc->y = 0;
                 crtc->enabled = false;
@@ -381,7 +381,7 @@ static int vmw_sou_crtc_set_config(struct drm_mode_set *set)
         connector->encoder = encoder;
         encoder->crtc = crtc;
         crtc->mode = *mode;
-        crtc->fb = fb;
+        crtc->primary->fb = fb;
         crtc->x = set->x;
         crtc->y = set->y;
         crtc->enabled = true;
@@ -572,5 +572,5 @@ void vmw_kms_screen_object_update_implicit_fb(struct vmw_private *dev_priv,
         BUG_ON(!sou->base.is_implicit);
 
         dev_priv->sou_priv->implicit_fb =
-                vmw_framebuffer_to_vfb(sou->base.crtc.fb);
+                vmw_framebuffer_to_vfb(sou->base.crtc.primary->fb);
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
index ee3856578a12..c1559eeaffe9 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_shader.c
@@ -449,7 +449,6 @@ int vmw_shader_define_ioctl(struct drm_device *dev, void *data,
         struct drm_vmw_shader_create_arg *arg =
                 (struct drm_vmw_shader_create_arg *)data;
         struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
-        struct vmw_master *vmaster = vmw_master(file_priv->master);
         struct vmw_dma_buffer *buffer = NULL;
         SVGA3dShaderType shader_type;
         int ret;
@@ -487,14 +486,14 @@ int vmw_shader_define_ioctl(struct drm_device *dev, void *data,
                 goto out_bad_arg;
         }
 
-        ret = ttm_read_lock(&vmaster->lock, true);
+        ret = ttm_read_lock(&dev_priv->reservation_sem, true);
         if (unlikely(ret != 0))
                 goto out_bad_arg;
 
         ret = vmw_shader_alloc(dev_priv, buffer, arg->size, arg->offset,
                                shader_type, tfile, &arg->shader_handle);
 
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
 out_bad_arg:
         vmw_dmabuf_unreference(&buffer);
         return ret;
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
index e7af580ab977..4ecdbf3e59da 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_surface.c
@@ -36,11 +36,13 @@
  * @base:           The TTM base object handling user-space visibility.
  * @srf:            The surface metadata.
  * @size:           TTM accounting size for the surface.
+ * @master:         master of the creating client. Used for security check.
  */
 struct vmw_user_surface {
         struct ttm_prime_object prime;
         struct vmw_surface srf;
         uint32_t size;
+        struct drm_master *master;
 };
 
 /**
@@ -624,6 +626,8 @@ static void vmw_user_surface_free(struct vmw_resource *res)
         struct vmw_private *dev_priv = srf->res.dev_priv;
         uint32_t size = user_srf->size;
 
+        if (user_srf->master)
+                drm_master_put(&user_srf->master);
         kfree(srf->offsets);
         kfree(srf->sizes);
         kfree(srf->snooper.image);
@@ -697,7 +701,6 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
         struct vmw_surface_offset *cur_offset;
         uint32_t num_sizes;
         uint32_t size;
-        struct vmw_master *vmaster = vmw_master(file_priv->master);
         const struct svga3d_surface_desc *desc;
 
         if (unlikely(vmw_user_surface_size == 0))
@@ -723,7 +726,7 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
                 return -EINVAL;
         }
 
-        ret = ttm_read_lock(&vmaster->lock, true);
+        ret = ttm_read_lock(&dev_priv->reservation_sem, true);
         if (unlikely(ret != 0))
                 return ret;
 
@@ -820,6 +823,8 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
 
         user_srf->prime.base.shareable = false;
         user_srf->prime.base.tfile = NULL;
+        if (drm_is_primary_client(file_priv))
+                user_srf->master = drm_master_get(file_priv->master);
 
         /**
          * From this point, the generic resource management functions
@@ -862,7 +867,7 @@ int vmw_surface_define_ioctl(struct drm_device *dev, void *data,
         rep->sid = user_srf->prime.base.hash.key;
         vmw_resource_unreference(&res);
 
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
         return 0;
 out_no_copy:
         kfree(srf->offsets);
@@ -873,7 +878,81 @@ out_no_sizes:
 out_no_user_srf:
         ttm_mem_global_free(vmw_mem_glob(dev_priv), size);
 out_unlock:
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
+        return ret;
+}
+
+
+static int
+vmw_surface_handle_reference(struct vmw_private *dev_priv,
+                             struct drm_file *file_priv,
+                             uint32_t u_handle,
+                             enum drm_vmw_handle_type handle_type,
+                             struct ttm_base_object **base_p)
+{
+        struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
+        struct vmw_user_surface *user_srf;
+        uint32_t handle;
+        struct ttm_base_object *base;
+        int ret;
+
+        if (handle_type == DRM_VMW_HANDLE_PRIME) {
+                ret = ttm_prime_fd_to_handle(tfile, u_handle, &handle);
+                if (unlikely(ret != 0))
+                        return ret;
+        } else {
+                if (unlikely(drm_is_render_client(file_priv))) {
+                        DRM_ERROR("Render client refused legacy "
+                                  "surface reference.\n");
+                        return -EACCES;
+                }
+                handle = u_handle;
+        }
+
+        ret = -EINVAL;
+        base = ttm_base_object_lookup_for_ref(dev_priv->tdev, handle);
+        if (unlikely(base == NULL)) {
+                DRM_ERROR("Could not find surface to reference.\n");
+                goto out_no_lookup;
+        }
+
+        if (unlikely(ttm_base_object_type(base) != VMW_RES_SURFACE)) {
+                DRM_ERROR("Referenced object is not a surface.\n");
+                goto out_bad_resource;
+        }
+
+        if (handle_type != DRM_VMW_HANDLE_PRIME) {
+                user_srf = container_of(base, struct vmw_user_surface,
+                                        prime.base);
+
+                /*
+                 * Make sure the surface creator has the same
+                 * authenticating master.
+                 */
+                if (drm_is_primary_client(file_priv) &&
+                    user_srf->master != file_priv->master) {
+                        DRM_ERROR("Trying to reference surface outside of"
+                                  " master domain.\n");
+                        ret = -EACCES;
+                        goto out_bad_resource;
+                }
+
+                ret = ttm_ref_object_add(tfile, base, TTM_REF_USAGE, NULL);
+                if (unlikely(ret != 0)) {
+                        DRM_ERROR("Could not add a reference to a surface.\n");
+                        goto out_bad_resource;
+                }
+        }
+
+        *base_p = base;
+        return 0;
+
+out_bad_resource:
+        ttm_base_object_unref(&base);
+out_no_lookup:
+        if (handle_type == DRM_VMW_HANDLE_PRIME)
+                (void) ttm_ref_object_base_unref(tfile, handle, TTM_REF_USAGE);
+
         return ret;
 }
 
@@ -898,27 +977,16 @@ int vmw_surface_reference_ioctl(struct drm_device *dev, void *data,
         struct vmw_user_surface *user_srf;
         struct drm_vmw_size __user *user_sizes;
         struct ttm_base_object *base;
-        int ret = -EINVAL;
-
-        base = ttm_base_object_lookup_for_ref(dev_priv->tdev, req->sid);
-        if (unlikely(base == NULL)) {
-                DRM_ERROR("Could not find surface to reference.\n");
-                return -EINVAL;
-        }
+        int ret;
 
-        if (unlikely(ttm_base_object_type(base) != VMW_RES_SURFACE))
-                goto out_bad_resource;
+        ret = vmw_surface_handle_reference(dev_priv, file_priv, req->sid,
+                                           req->handle_type, &base);
+        if (unlikely(ret != 0))
+                return ret;
 
         user_srf = container_of(base, struct vmw_user_surface, prime.base);
         srf = &user_srf->srf;
 
-        ret = ttm_ref_object_add(tfile, &user_srf->prime.base,
-                                 TTM_REF_USAGE, NULL);
-        if (unlikely(ret != 0)) {
-                DRM_ERROR("Could not add a reference to a surface.\n");
-                goto out_no_reference;
-        }
-
         rep->flags = srf->flags;
         rep->format = srf->format;
         memcpy(rep->mip_levels, srf->mip_levels, sizeof(srf->mip_levels));
@@ -931,10 +999,10 @@ int vmw_surface_reference_ioctl(struct drm_device *dev, void *data,
         if (unlikely(ret != 0)) {
                 DRM_ERROR("copy_to_user failed %p %u\n",
                           user_sizes, srf->num_sizes);
+                ttm_ref_object_base_unref(tfile, base->hash.key, TTM_REF_USAGE);
                 ret = -EFAULT;
         }
-out_bad_resource:
-out_no_reference:
+
         ttm_base_object_unref(&base);
 
         return ret;
@@ -1173,7 +1241,6 @@ int vmw_gb_surface_define_ioctl(struct drm_device *dev, void *data,
         struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
         int ret;
         uint32_t size;
-        struct vmw_master *vmaster = vmw_master(file_priv->master);
         const struct svga3d_surface_desc *desc;
         uint32_t backup_handle;
 
@@ -1189,7 +1256,7 @@ int vmw_gb_surface_define_ioctl(struct drm_device *dev, void *data,
                 return -EINVAL;
         }
 
-        ret = ttm_read_lock(&vmaster->lock, true);
+        ret = ttm_read_lock(&dev_priv->reservation_sem, true);
         if (unlikely(ret != 0))
                 return ret;
 
@@ -1228,6 +1295,8 @@ int vmw_gb_surface_define_ioctl(struct drm_device *dev, void *data,
 
         user_srf->prime.base.shareable = false;
         user_srf->prime.base.tfile = NULL;
+        if (drm_is_primary_client(file_priv))
+                user_srf->master = drm_master_get(file_priv->master);
 
         /**
          * From this point, the generic resource management functions
@@ -1283,12 +1352,12 @@ int vmw_gb_surface_define_ioctl(struct drm_device *dev, void *data,
 
         vmw_resource_unreference(&res);
 
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
         return 0;
 out_no_user_srf:
         ttm_mem_global_free(vmw_mem_glob(dev_priv), size);
 out_unlock:
-        ttm_read_unlock(&vmaster->lock);
+        ttm_read_unlock(&dev_priv->reservation_sem);
         return ret;
 }
 
@@ -1315,14 +1384,10 @@ int vmw_gb_surface_reference_ioctl(struct drm_device *dev, void *data,
         uint32_t backup_handle;
         int ret = -EINVAL;
 
-        base = ttm_base_object_lookup_for_ref(dev_priv->tdev, req->sid);
-        if (unlikely(base == NULL)) {
-                DRM_ERROR("Could not find surface to reference.\n");
-                return -EINVAL;
-        }
-
-        if (unlikely(ttm_base_object_type(base) != VMW_RES_SURFACE))
-                goto out_bad_resource;
+        ret = vmw_surface_handle_reference(dev_priv, file_priv, req->sid,
+                                           req->handle_type, &base);
+        if (unlikely(ret != 0))
+                return ret;
 
         user_srf = container_of(base, struct vmw_user_surface, prime.base);
         srf = &user_srf->srf;
@@ -1331,13 +1396,6 @@ int vmw_gb_surface_reference_ioctl(struct drm_device *dev, void *data,
                 goto out_bad_resource;
         }
 
-        ret = ttm_ref_object_add(tfile, &user_srf->prime.base,
-                                 TTM_REF_USAGE, NULL);
-        if (unlikely(ret != 0)) {
-                DRM_ERROR("Could not add a reference to a GB surface.\n");
-                goto out_bad_resource;
-        }
-
         mutex_lock(&dev_priv->cmdbuf_mutex); /* Protect res->backup */
         ret = vmw_user_dmabuf_reference(tfile, srf->res.backup,
                                         &backup_handle);
@@ -1346,8 +1404,7 @@ int vmw_gb_surface_reference_ioctl(struct drm_device *dev, void *data,
         if (unlikely(ret != 0)) {
                 DRM_ERROR("Could not add a reference to a GB surface "
                           "backup buffer.\n");
-                (void) ttm_ref_object_base_unref(vmw_fpriv(file_priv)->tfile,
-                                                 req->sid,
+                (void) ttm_ref_object_base_unref(tfile, base->hash.key,
                                                  TTM_REF_USAGE);
                 goto out_bad_resource;
         }
diff --git a/drivers/gpu/host1x/syncpt.c b/drivers/gpu/host1x/syncpt.c
index bfb09d802abd..b10550ee1d89 100644
--- a/drivers/gpu/host1x/syncpt.c
+++ b/drivers/gpu/host1x/syncpt.c
@@ -102,6 +102,7 @@ u32 host1x_syncpt_incr_max(struct host1x_syncpt *sp, u32 incrs)
 {
         return (u32)atomic_add_return(incrs, &sp->max_val);
 }
+EXPORT_SYMBOL(host1x_syncpt_incr_max);
 
  /*
  * Write cached syncpoint and waitbase values to hardware.
diff --git a/drivers/staging/imx-drm/ipuv3-crtc.c b/drivers/staging/imx-drm/ipuv3-crtc.c
index a8d017854615..c48f640db006 100644
--- a/drivers/staging/imx-drm/ipuv3-crtc.c
+++ b/drivers/staging/imx-drm/ipuv3-crtc.c
@@ -121,7 +121,7 @@ static int ipu_page_flip(struct drm_crtc *crtc,
 
         ipu_crtc->newfb = fb;
         ipu_crtc->page_flip_event = event;
-        crtc->fb = fb;
+        crtc->primary->fb = fb;
 
         return 0;
 }
@@ -193,7 +193,7 @@ static int ipu_crtc_mode_set(struct drm_crtc *crtc,
                 return ret;
         }
 
-        return ipu_plane_mode_set(ipu_crtc->plane[0], crtc, mode, crtc->fb,
+        return ipu_plane_mode_set(ipu_crtc->plane[0], crtc, mode, crtc->primary->fb,
                                   0, 0, mode->hdisplay, mode->vdisplay,
                                   x, y, mode->hdisplay, mode->vdisplay);
 }
@@ -219,7 +219,7 @@ static irqreturn_t ipu_irq_handler(int irq, void *dev_id)
 
         if (ipu_crtc->newfb) {
                 ipu_crtc->newfb = NULL;
-                ipu_plane_set_base(ipu_crtc->plane[0], ipu_crtc->base.fb,
+                ipu_plane_set_base(ipu_crtc->plane[0], ipu_crtc->base.primary->fb,
                                 ipu_crtc->plane[0]->x, ipu_crtc->plane[0]->y);
                 ipu_crtc_handle_pageflip(ipu_crtc);
         }
diff --git a/drivers/staging/imx-drm/ipuv3-plane.c b/drivers/staging/imx-drm/ipuv3-plane.c
index b0c9b6ce4854..27a8d735dae0 100644
--- a/drivers/staging/imx-drm/ipuv3-plane.c
+++ b/drivers/staging/imx-drm/ipuv3-plane.c
@@ -68,7 +68,7 @@ int ipu_plane_set_base(struct ipu_plane *ipu_plane, struct drm_framebuffer *fb,
 
         cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
         if (!cma_obj) {
-                DRM_LOG_KMS("entry is null.\n");
+                DRM_DEBUG_KMS("entry is null.\n");
                 return -EFAULT;
         }
 
diff --git a/drivers/video/exynos/Kconfig b/drivers/video/exynos/Kconfig
index eb6f2b059821..fcf2d48ac6d1 100644
--- a/drivers/video/exynos/Kconfig
+++ b/drivers/video/exynos/Kconfig
@@ -29,11 +29,4 @@ config EXYNOS_LCD_S6E8AX0
           If you have an S6E8AX0 MIPI AMOLED LCD Panel, say Y to enable its
           LCD control driver.
 
-config EXYNOS_DP
-        bool "EXYNOS DP driver support"
-        depends on ARCH_EXYNOS
-        default n
-        help
-          This enables support for DP device.
-
 endif # EXYNOS_VIDEO
diff --git a/drivers/video/exynos/Makefile b/drivers/video/exynos/Makefile
index ec7772e452a9..b5b1bd228abb 100644
--- a/drivers/video/exynos/Makefile
+++ b/drivers/video/exynos/Makefile
@@ -5,4 +5,3 @@
 obj-$(CONFIG_EXYNOS_MIPI_DSI)           += exynos_mipi_dsi.o exynos_mipi_dsi_common.o \
                                         exynos_mipi_dsi_lowlevel.o
 obj-$(CONFIG_EXYNOS_LCD_S6E8AX0)        += s6e8ax0.o
-obj-$(CONFIG_EXYNOS_DP)                 += exynos_dp_core.o exynos_dp_reg.o